{"repo": "abrehamgebremedhin/smartstudy", "files": {"backend/scripts/loadtest_ws.py": {"271ca9e7b637c62145a768bf5f572225d3a4d31a": {"t": 1785745445.2023993, "n": 5054, "h": "ccdeb61af0f1494a", "c": "#!/usr/bin/env python3\n\"\"\"End-to-end WS load test — REAL generations (this bills DeepSeek + runs Ollama).\n\nOpens `--concurrency` real WebSocket connections to /api/ws/generate/mcq at once,\neach asking for a fresh generation, and measures what an actual user feels:\ntime-to-result, success rate, and how badly latency degrades once the burst\nexceeds the server's 8 worker slots (the overflow queues).\n\nUnlike loadtest_queue.py this exercises the whole stack — auth, WS framing,\nOllama embedding retrieval, the DeepSeek call — so the numbers are real and so\nis the cost. Keep --num-questions small and start with a low --concurrency.\n\nPrereqs: the API server running (uvicorn), Ollama up, DEEPSEEK_API_KEY in .env.\n\nUsage\n-----\n    # 5 users at once, 3 questions each:\n    uv run python scripts/loadtest_ws.py --concurrency 5 --num-questions 3\n\n    # push past the 8-worker cap to see queueing:\n    uv run python scripts/loadtest_ws.py --concurrency 16 --num-questions 3\n\"\"\"\nimport argparse\nimport asyncio\nimport statistics\nimport sys\nimport time\nfrom pathlib import Path\n\nimport websockets\n\nsys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # backend/ on path\n\nfrom app.auth.tokens import create_app_token  # noqa: E402\n\nPAYLOAD = {\"subject\": \"physics\", \"grade\": 11, \"unit\": \"1\", \"difficulty\": \"medium\"}\n\n\nasync def _one(url: str, payload: dict, timeout: float) -> dict:\n    \"\"\"Run one generation over a fresh WS; return its outcome + timing.\"\"\"\n    t0 = time.monotonic()\n    stages = 0\n    try:\n        async with websockets.connect(url, max_size=None, open_timeout=15) as ws:\n            await ws.send(_dumps(payload))\n            while True:\n                raw = await asyncio.wait_for(ws.recv(), timeout=timeout)\n                msg = _loads(raw)\n                kind = msg.get(\"type\")\n                if kind == \"progress\":\n                    stages += 1\n                elif kind == \"result\":\n                    return {\"ok\": True, \"secs\": time.monotonic() - t0, \"stages\": stages,\n                            \"cache\": msg[\"data\"].get(\"was_cache_hit\")}\n                elif kind == \"error\":\n                    return {\"ok\": False, \"secs\": time.monotonic() - t0,\n                            \"err\": f\"{msg.get('code')}: {msg.get('detail')}\"}\n    except Exception as e:  # noqa: BLE001\n        return {\"ok\": False, \"secs\": time.monotonic() - t0, \"err\": f\"{type(e).__name__}: {e}\"}\n\n\ndef _url(host: str, i: int) -> str:\n    # Distinct user per connection — concurrent logins of the *same* new google_id\n    # race in get_or_create_user, which would mask the real capacity numbers.\n    token = create_app_token({\"sub\": f\"loadtest-{i}\", \"email\": f\"loadtest-{i}@example.com\"})\n    return f\"{host}/api/ws/generate/mcq?token={token}\"\n\n\nasync def main(host: str, concurrency: int, num_q: int, timeout: float) -> None:\n    payload = {**PAYLOAD, \"num_questions\": num_q}\n    print(f\"Firing {concurrency} concurrent generations, {num_q} questions each -> {host}\")\n    print(\"(server caps concurrent generation at MAX_WORKERS=8; the rest queue)\\n\")\n\n    t0 = time.monotonic()\n    results = await asyncio.gather(*(_one(_url(host, i), payload, timeout) for i in range(concurrency)))\n    wall = time.monotonic() - t0\n\n    ok = [r for r in results if r[\"ok\"]]\n    bad = [r for r in results if not r[\"ok\"]]\n    lat = sorted(r[\"secs\"] for r in ok)\n\n    print(f\"Wall time:     {wall:.1f}s for {concurrency} requests\")\n    print(f\"Succeeded:     {len(ok)}/{concurrency}\" + (f\"   (cache hits: {sum(bool(r.get('cache')) for r in ok)})\" if ok else \"\"))\n    if lat:\n        print(f\"Time-to-result p50/p95/max: {_pct(lat, 50):.1f}s / {_pct(lat, 95):.1f}s / {lat[-1]:.1f}s\")\n        print(f\"Throughput:    {len(ok) / wall:.2f} gen/s ({len(ok) / wall * 60:.0f}/min)\")\n    if bad:\n        print(f\"\\nFailed {len(bad)}:\")\n        for r in bad[:10]:\n            print(f\"  {r['secs']:.1f}s  {r['err']}\")\n\n\ndef _pct(values: list[float], p: int) -> float:\n    if len(values) <= 1:\n        return values[0] if values else 0.0\n    return statistics.quantiles(values, n=100)[p - 1]\n\n\n# orjson if present (server uses it), else stdlib — keep this script dependency-free.\ntry:\n    import orjson\n\n    def _dumps(o: dict) -> str: return orjson.dumps(o).decode()\n    def _loads(s): return orjson.loads(s)\nexcept ImportError:\n    import json\n\n    def _dumps(o: dict) -> str: return json.dumps(o)\n    def _loads(s): return json.loads(s)\n\n\nif __name__ == \"__main__\":\n    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)\n    ap.add_argument(\"--host\", default=\"ws://localhost:8000\", help=\"server ws:// base URL\")\n    ap.add_argument(\"--concurrency\", type=int, default=5, help=\"simultaneous WS generations\")\n    ap.add_argument(\"--num-questions\", type=int, default=3, help=\"questions per generation (cost knob)\")\n    ap.add_argument(\"--timeout\", type=float, default=120.0, help=\"per-request seconds before giving up\")\n    args = ap.parse_args()\n    asyncio.run(main(args.host, args.concurrency, args.num_questions, args.timeout))\n"}, "cf06aa079676bf859da585ee5bcdb9060037f6cd": {"t": 1785753311.4328923, "n": 5054, "h": "ccdeb61af0f1494a", "c": "#!/usr/bin/env python3\n\"\"\"End-to-end WS load test — REAL generations (this bills DeepSeek + runs Ollama).\n\nOpens `--concurrency` real WebSocket connections to /api/ws/generate/mcq at once,\neach asking for a fresh generation, and measures what an actual user feels:\ntime-to-result, success rate, and how badly latency degrades once the burst\nexceeds the server's 8 worker slots (the overflow queues).\n\nUnlike loadtest_queue.py this exercises the whole stack — auth, WS framing,\nOllama embedding retrieval, the DeepSeek call — so the numbers are real and so\nis the cost. Keep --num-questions small and start with a low --concurrency.\n\nPrereqs: the API server running (uvicorn), Ollama up, DEEPSEEK_API_KEY in .env.\n\nUsage\n-----\n    # 5 users at once, 3 questions each:\n    uv run python scripts/loadtest_ws.py --concurrency 5 --num-questions 3\n\n    # push past the 8-worker cap to see queueing:\n    uv run python scripts/loadtest_ws.py --concurrency 16 --num-questions 3\n\"\"\"\nimport argparse\nimport asyncio\nimport statistics\nimport sys\nimport time\nfrom pathlib import Path\n\nimport websockets\n\nsys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # backend/ on path\n\nfrom app.auth.tokens import create_app_token  # noqa: E402\n\nPAYLOAD = {\"subject\": \"physics\", \"grade\": 11, \"unit\": \"1\", \"difficulty\": \"medium\"}\n\n\nasync def _one(url: str, payload: dict, timeout: float) -> dict:\n    \"\"\"Run one generation over a fresh WS; return its outcome + timing.\"\"\"\n    t0 = time.monotonic()\n    stages = 0\n    try:\n        async with websockets.connect(url, max_size=None, open_timeout=15) as ws:\n            await ws.send(_dumps(payload))\n            while True:\n                raw = await asyncio.wait_for(ws.recv(), timeout=timeout)\n                msg = _loads(raw)\n                kind = msg.get(\"type\")\n                if kind == \"progress\":\n                    stages += 1\n                elif kind == \"result\":\n                    return {\"ok\": True, \"secs\": time.monotonic() - t0, \"stages\": stages,\n                            \"cache\": msg[\"data\"].get(\"was_cache_hit\")}\n                elif kind == \"error\":\n                    return {\"ok\": False, \"secs\": time.monotonic() - t0,\n                            \"err\": f\"{msg.get('code')}: {msg.get('detail')}\"}\n    except Exception as e:  # noqa: BLE001\n        return {\"ok\": False, \"secs\": time.monotonic() - t0, \"err\": f\"{type(e).__name__}: {e}\"}\n\n\ndef _url(host: str, i: int) -> str:\n    # Distinct user per connection — concurrent logins of the *same* new google_id\n    # race in get_or_create_user, which would mask the real capacity numbers.\n    token = create_app_token({\"sub\": f\"loadtest-{i}\", \"email\": f\"loadtest-{i}@example.com\"})\n    return f\"{host}/api/ws/generate/mcq?token={token}\"\n\n\nasync def main(host: str, concurrency: int, num_q: int, timeout: float) -> None:\n    payload = {**PAYLOAD, \"num_questions\": num_q}\n    print(f\"Firing {concurrency} concurrent generations, {num_q} questions each -> {host}\")\n    print(\"(server caps concurrent generation at MAX_WORKERS=8; the rest queue)\\n\")\n\n    t0 = time.monotonic()\n    results = await asyncio.gather(*(_one(_url(host, i), payload, timeout) for i in range(concurrency)))\n    wall = time.monotonic() - t0\n\n    ok = [r for r in results if r[\"ok\"]]\n    bad = [r for r in results if not r[\"ok\"]]\n    lat = sorted(r[\"secs\"] for r in ok)\n\n    print(f\"Wall time:     {wall:.1f}s for {concurrency} requests\")\n    print(f\"Succeeded:     {len(ok)}/{concurrency}\" + (f\"   (cache hits: {sum(bool(r.get('cache')) for r in ok)})\" if ok else \"\"))\n    if lat:\n        print(f\"Time-to-result p50/p95/max: {_pct(lat, 50):.1f}s / {_pct(lat, 95):.1f}s / {lat[-1]:.1f}s\")\n        print(f\"Throughput:    {len(ok) / wall:.2f} gen/s ({len(ok) / wall * 60:.0f}/min)\")\n    if bad:\n        print(f\"\\nFailed {len(bad)}:\")\n        for r in bad[:10]:\n            print(f\"  {r['secs']:.1f}s  {r['err']}\")\n\n\ndef _pct(values: list[float], p: int) -> float:\n    if len(values) <= 1:\n        return values[0] if values else 0.0\n    return statistics.quantiles(values, n=100)[p - 1]\n\n\n# orjson if present (server uses it), else stdlib — keep this script dependency-free.\ntry:\n    import orjson\n\n    def _dumps(o: dict) -> str: return orjson.dumps(o).decode()\n    def _loads(s): return orjson.loads(s)\nexcept ImportError:\n    import json\n\n    def _dumps(o: dict) -> str: return json.dumps(o)\n    def _loads(s): return json.loads(s)\n\n\nif __name__ == \"__main__\":\n    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)\n    ap.add_argument(\"--host\", default=\"ws://localhost:8000\", help=\"server ws:// base URL\")\n    ap.add_argument(\"--concurrency\", type=int, default=5, help=\"simultaneous WS generations\")\n    ap.add_argument(\"--num-questions\", type=int, default=3, help=\"questions per generation (cost knob)\")\n    ap.add_argument(\"--timeout\", type=float, default=120.0, help=\"per-request seconds before giving up\")\n    args = ap.parse_args()\n    asyncio.run(main(args.host, args.concurrency, args.num_questions, args.timeout))\n"}}}}