1 line
20 KiB
JSON
1 line
20 KiB
JSON
{"repo": "billkst/rivalradar", "files": {"tests/test_llm_limits.py": {"c1f6bf90376febfe9fe0db5e3365c14f4bcef5c2": {"t": 1785747498.9151108, "n": 7802, "h": "cf6f4710e84ef44f", "c": "\"\"\"厂商输出上限钳制:让「模型可插拔」在真 run 上真的成立。\n\n背景(这批测试防的是什么病):`structured_call` 默认按 131072 要输出额度 —— 那是**火山方舟\n端点的实测硬上限**,写死在 harness 里。BYOK 换成 DeepSeek/OpenAI 后:\n - `/llm/ping` 原先发 max_tokens=1 → **必过**;\n - 真 run 的第一次 structured_call 发 131072 → 400 → 被当网络抖动重试 3 次 → 死 run。\n「测试通过、真跑就死」,而且用户会误以为是厂商不兼容。\n\"\"\"\nfrom __future__ import annotations\n\nfrom types import SimpleNamespace\n\nimport httpx\nimport openai\nimport pytest\nfrom fastapi.testclient import TestClient\n\nfrom rivalradar.api.app import create_app\nfrom rivalradar.llm.limits import cap_client\nfrom rivalradar.llm.structured import _DEFAULT_MAX_TOKENS, StructuredCallError, structured_call\nfrom rivalradar.schema.models import EvidenceRef\n\n\n@pytest.fixture()\ndef db_path(tmp_path):\n return str(tmp_path / \"t.db\")\n\n\nclass _Recorder:\n \"\"\"记录每次 create 的 kwargs;可选择性抛异常。\"\"\"\n\n def __init__(self, err: Exception | None = None):\n self.calls: list[dict] = []\n self._err = err\n\n def create(self, **kwargs):\n self.calls.append(kwargs)\n if self._err is not None:\n raise self._err\n return SimpleNamespace(choices=[], usage=None)\n\n\ndef _client(rec: _Recorder, api_key: str = \"sk-secret\"):\n return SimpleNamespace(chat=SimpleNamespace(completions=rec), api_key=api_key)\n\n\n# ── 钳制本身 ─────────────────────────────────────────────────────────────────\n\ndef test_no_cap_returns_client_unchanged():\n \"\"\"env(方舟)路径不声明上限 → 原样返回,行为与改动前逐字节一致。\"\"\"\n c = _client(_Recorder())\n assert cap_client(c, None) is c\n\n\ndef test_cap_clamps_oversized_request():\n \"\"\"131072(方舟硬上限)对 DeepSeek 过大 → 被钳到厂商上限,而不是发出去挨 400。\"\"\"\n rec = _Recorder()\n cap_client(_client(rec), 8192).chat.completions.create(\n model=\"deepseek-chat\", messages=[], max_tokens=_DEFAULT_MAX_TOKENS)\n assert rec.calls[0][\"max_tokens\"] == 8192\n\n\ndef test_cap_does_not_inflate_smaller_request():\n \"\"\"取 min 而非覆盖:ping 若只要 1 个 token,不该被抬成上限值。\"\"\"\n rec = _Recorder()\n cap_client(_client(rec), 8192).chat.completions.create(\n model=\"m\", messages=[], max_tokens=1)\n assert rec.calls[0][\"max_tokens\"] == 1\n\n\ndef test_cap_injects_when_max_tokens_absent():\n \"\"\"调用方没传 max_tokens(如 writer 的流式 insight 草稿)→ cap 补上上限,让「声明了\n 上限就对所有调用生效」的不变量真的成立(评审 F4)。补的是厂商允许的最大值,不截断。\"\"\"\n rec = _Recorder()\n cap_client(_client(rec), 8192).chat.completions.create(model=\"m\", messages=[])\n assert rec.calls[0][\"max_tokens\"] == 8192\n\n\ndef test_cap_wrapper_passes_through_api_key_for_redaction():\n \"\"\"脱敏门闩:structured.py 靠 getattr(client,'api_key') 取 key 打码,包装层不能吞掉它。\"\"\"\n assert cap_client(_client(_Recorder(), api_key=\"sk-live-9\"), 8192).api_key == \"sk-live-9\"\n\n\n# ── 400 快速失败(不再白烧 3 次调用)────────────────────────────────────────\n\ndef _bad_request(msg: str = \"max_tokens must be <= 8192\") -> openai.BadRequestError:\n req = httpx.Request(\"POST\", \"https://api.deepseek.com/v1/chat/completions\")\n return openai.BadRequestError(msg, response=httpx.Response(400, request=req), body=None)\n\n\ndef test_structured_call_fails_fast_on_400_without_retrying():\n \"\"\"400 是**确定性**错误:同样的请求重试必然同样失败。原先它落进 APIError 分支被重试 3 次\n —— 白烧 3 次真金白银的调用(每次还叠一个 90s timeout),还把「我们发错了参数」的诊断\n 埋进超时噪声里。必须一次就停,并把厂商的 400 原文(脱敏后)带出来。\"\"\"\n rec = _Recorder(err=_bad_request())\n with pytest.raises(StructuredCallError) as ei:\n structured_call(EvidenceRef, [{\"role\": \"user\", \"content\": \"x\"}],\n client=_client(rec), model=\"deepseek-chat\")\n assert len(rec.calls) == 1 # 只调了一次 = 没有重复计费\n assert \"不重试\" in str(ei.value)\n assert \"max_tokens must be <= 8192\" in str(ei.value) # 厂商原文可见 = 一眼看懂病因\n\n\ndef test_structured_call_400_message_is_redacted():\n \"\"\"400 的 body 可能回显 key(某些厂商会把整个请求回显)→ 必须脱敏后再进异常消息。\"\"\"\n rec = _Recorder(err=_bad_request(\"bad request with key sk-leak-abcdefgh12345\"))\n with pytest.raises(StructuredCallError) as ei:\n structured_call(EvidenceRef, [{\"role\": \"user\", \"content\": \"x\"}],\n client=_client(rec, api_key=\"sk-leak-abcdefgh12345\"),\n model=\"m\")\n assert \"sk-leak-abcdefgh12345\" not in str(ei.value)\n assert \"***\" in str(ei.value)\n\n\ndef test_structured_call_400_does_not_leak_key_via_exception_chain():\n \"\"\"外层消息脱敏还不够:`from err` 会把**未脱敏**的原始 BadRequestError 挂到 __cause__ 上,\n 调用方一句 logger.warning(..., exc_info=True)(analyst._safe_extract 正是如此)就把整条\n traceback(含 __cause__ 的 str)打进日志 → key 从异常链泄漏(评审 P1)。必须 from None 断链。\"\"\"\n import traceback\n\n leak = \"sk-leak-abcdefgh12345\"\n rec = _Recorder(err=_bad_request(f\"provider echoed your key {leak}\"))\n with pytest.raises(StructuredCallError) as ei:\n structured_call(EvidenceRef, [{\"role\": \"user\", \"content\": \"x\"}],\n client=_client(rec, api_key=leak), model=\"m\")\n exc = ei.value\n # from None:__cause__ 断开,__context__ 被抑制,exc_info 打印时不会带出原始异常\n assert exc.__cause__ is None\n assert exc.__suppress_context__ is True\n rendered = \"\".join(traceback.format_exception(type(exc), exc, exc.__traceback__))\n assert leak not in rendered, \"key 从异常链(exc_info 渲染)泄漏\"\n\n\n# ── 端到端:头 → 钳制 → ping 发的是真 run 的那个值 ───────────────────────────\n\n_BYOK = {\n \"X-LLM-Base-URL\": \"https://api.deepseek.com/v1\",\n \"X-LLM-API-Key\": \"sk-user-key\",\n \"X-LLM-Model\": \"deepseek-chat\",\n}\n\n\ndef test_ping_sends_capped_max_tokens_matching_real_run(db_path, monkeypatch):\n \"\"\"整条链的验收:声明上限 8192 → ping 实际发出 8192(= min(131072, 8192))。\n 这正是真 run 的 structured_call 会发的值 —— **连通性测试与真 run 逐字节一致**。\"\"\"\n rec = _Recorder()\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(rec))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n r = client.post(\"/llm/ping\", headers={**_BYOK, \"X-LLM-Max-Tokens\": \"8192\"})\n\n assert r.status_code == 200 and r.json()[\"ok\"] is True\n assert rec.calls[0][\"max_tokens\"] == 8192\n\n\ndef test_ping_without_cap_sends_full_default(db_path, monkeypatch):\n \"\"\"不声明上限(老配置 / 方舟)→ 发默认 131072,与改动前一致。\"\"\"\n rec = _Recorder()\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(rec))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n client.post(\"/llm/ping\", headers=_BYOK)\n\n assert rec.calls[0][\"max_tokens\"] == _DEFAULT_MAX_TOKENS\n\n\ndef test_ping_classifies_400_as_bad_request(db_path, monkeypatch):\n \"\"\"上限填得比厂商允许的大 → ping 当场分类 bad_request,前端渲染「按厂商文档调小」。\n 这就是「点一下按钮就知道」而不是「跑 7 分钟死 run 才知道」。\"\"\"\n rec = _Recorder(err=_bad_request())\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(rec))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n body = client.post(\"/llm/ping\", headers={**_BYOK, \"X-LLM-Max-Tokens\": \"999999\"}).json()\n\n assert body[\"ok\"] is False\n assert body[\"error_type\"] == \"bad_request\"\n assert \"max_tokens\" in body[\"detail\"]\n\n\n@pytest.mark.parametrize(\"bad\", [\"abc\", \"0\", \"-1\", \"1048577\", \"1.5\"])\ndef test_invalid_max_tokens_header_is_422_not_silently_ignored(db_path, monkeypatch, bad):\n \"\"\"非法值必须 422。静默忽略最坏 —— 用户以为设了上限,真 run 仍按 131072 发出去挨 400,\n 排查成本极高(界面显示「已配置 8192」,实际发的是 131072)。\"\"\"\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(_Recorder()))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n r = client.post(\"/llm/ping\", headers={**_BYOK, \"X-LLM-Max-Tokens\": bad})\n\n assert r.status_code == 422\n assert \"X-LLM-Max-Tokens\" in r.json()[\"detail\"]\n\n\ndef test_max_tokens_header_alone_does_not_trigger_all_or_nothing(db_path):\n \"\"\"第 4 个头是**可选**的,不进「全有或全无」契约:只给它(无三头)仍应走 env fallback\n 路径的 503,而不是 422「BYOK 需三头齐全」—— 否则老前端一发这个头就全挂。\"\"\"\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n r = client.post(\"/llm/ping\", headers={\"X-LLM-Max-Tokens\": \"8192\"})\n assert r.status_code == 200\n assert r.json()[\"error_type\"] == \"unconfigured\"\n"}, "5f5d7d62150c982d9635a8ba51df034aaa5d6fed": {"t": 1785773613.6571867, "n": 7802, "h": "cf6f4710e84ef44f", "c": "\"\"\"厂商输出上限钳制:让「模型可插拔」在真 run 上真的成立。\n\n背景(这批测试防的是什么病):`structured_call` 默认按 131072 要输出额度 —— 那是**火山方舟\n端点的实测硬上限**,写死在 harness 里。BYOK 换成 DeepSeek/OpenAI 后:\n - `/llm/ping` 原先发 max_tokens=1 → **必过**;\n - 真 run 的第一次 structured_call 发 131072 → 400 → 被当网络抖动重试 3 次 → 死 run。\n「测试通过、真跑就死」,而且用户会误以为是厂商不兼容。\n\"\"\"\nfrom __future__ import annotations\n\nfrom types import SimpleNamespace\n\nimport httpx\nimport openai\nimport pytest\nfrom fastapi.testclient import TestClient\n\nfrom rivalradar.api.app import create_app\nfrom rivalradar.llm.limits import cap_client\nfrom rivalradar.llm.structured import _DEFAULT_MAX_TOKENS, StructuredCallError, structured_call\nfrom rivalradar.schema.models import EvidenceRef\n\n\n@pytest.fixture()\ndef db_path(tmp_path):\n return str(tmp_path / \"t.db\")\n\n\nclass _Recorder:\n \"\"\"记录每次 create 的 kwargs;可选择性抛异常。\"\"\"\n\n def __init__(self, err: Exception | None = None):\n self.calls: list[dict] = []\n self._err = err\n\n def create(self, **kwargs):\n self.calls.append(kwargs)\n if self._err is not None:\n raise self._err\n return SimpleNamespace(choices=[], usage=None)\n\n\ndef _client(rec: _Recorder, api_key: str = \"sk-secret\"):\n return SimpleNamespace(chat=SimpleNamespace(completions=rec), api_key=api_key)\n\n\n# ── 钳制本身 ─────────────────────────────────────────────────────────────────\n\ndef test_no_cap_returns_client_unchanged():\n \"\"\"env(方舟)路径不声明上限 → 原样返回,行为与改动前逐字节一致。\"\"\"\n c = _client(_Recorder())\n assert cap_client(c, None) is c\n\n\ndef test_cap_clamps_oversized_request():\n \"\"\"131072(方舟硬上限)对 DeepSeek 过大 → 被钳到厂商上限,而不是发出去挨 400。\"\"\"\n rec = _Recorder()\n cap_client(_client(rec), 8192).chat.completions.create(\n model=\"deepseek-chat\", messages=[], max_tokens=_DEFAULT_MAX_TOKENS)\n assert rec.calls[0][\"max_tokens\"] == 8192\n\n\ndef test_cap_does_not_inflate_smaller_request():\n \"\"\"取 min 而非覆盖:ping 若只要 1 个 token,不该被抬成上限值。\"\"\"\n rec = _Recorder()\n cap_client(_client(rec), 8192).chat.completions.create(\n model=\"m\", messages=[], max_tokens=1)\n assert rec.calls[0][\"max_tokens\"] == 1\n\n\ndef test_cap_injects_when_max_tokens_absent():\n \"\"\"调用方没传 max_tokens(如 writer 的流式 insight 草稿)→ cap 补上上限,让「声明了\n 上限就对所有调用生效」的不变量真的成立(评审 F4)。补的是厂商允许的最大值,不截断。\"\"\"\n rec = _Recorder()\n cap_client(_client(rec), 8192).chat.completions.create(model=\"m\", messages=[])\n assert rec.calls[0][\"max_tokens\"] == 8192\n\n\ndef test_cap_wrapper_passes_through_api_key_for_redaction():\n \"\"\"脱敏门闩:structured.py 靠 getattr(client,'api_key') 取 key 打码,包装层不能吞掉它。\"\"\"\n assert cap_client(_client(_Recorder(), api_key=\"sk-live-9\"), 8192).api_key == \"sk-live-9\"\n\n\n# ── 400 快速失败(不再白烧 3 次调用)────────────────────────────────────────\n\ndef _bad_request(msg: str = \"max_tokens must be <= 8192\") -> openai.BadRequestError:\n req = httpx.Request(\"POST\", \"https://api.deepseek.com/v1/chat/completions\")\n return openai.BadRequestError(msg, response=httpx.Response(400, request=req), body=None)\n\n\ndef test_structured_call_fails_fast_on_400_without_retrying():\n \"\"\"400 是**确定性**错误:同样的请求重试必然同样失败。原先它落进 APIError 分支被重试 3 次\n —— 白烧 3 次真金白银的调用(每次还叠一个 90s timeout),还把「我们发错了参数」的诊断\n 埋进超时噪声里。必须一次就停,并把厂商的 400 原文(脱敏后)带出来。\"\"\"\n rec = _Recorder(err=_bad_request())\n with pytest.raises(StructuredCallError) as ei:\n structured_call(EvidenceRef, [{\"role\": \"user\", \"content\": \"x\"}],\n client=_client(rec), model=\"deepseek-chat\")\n assert len(rec.calls) == 1 # 只调了一次 = 没有重复计费\n assert \"不重试\" in str(ei.value)\n assert \"max_tokens must be <= 8192\" in str(ei.value) # 厂商原文可见 = 一眼看懂病因\n\n\ndef test_structured_call_400_message_is_redacted():\n \"\"\"400 的 body 可能回显 key(某些厂商会把整个请求回显)→ 必须脱敏后再进异常消息。\"\"\"\n rec = _Recorder(err=_bad_request(\"bad request with key sk-leak-abcdefgh12345\"))\n with pytest.raises(StructuredCallError) as ei:\n structured_call(EvidenceRef, [{\"role\": \"user\", \"content\": \"x\"}],\n client=_client(rec, api_key=\"sk-leak-abcdefgh12345\"),\n model=\"m\")\n assert \"sk-leak-abcdefgh12345\" not in str(ei.value)\n assert \"***\" in str(ei.value)\n\n\ndef test_structured_call_400_does_not_leak_key_via_exception_chain():\n \"\"\"外层消息脱敏还不够:`from err` 会把**未脱敏**的原始 BadRequestError 挂到 __cause__ 上,\n 调用方一句 logger.warning(..., exc_info=True)(analyst._safe_extract 正是如此)就把整条\n traceback(含 __cause__ 的 str)打进日志 → key 从异常链泄漏(评审 P1)。必须 from None 断链。\"\"\"\n import traceback\n\n leak = \"sk-leak-abcdefgh12345\"\n rec = _Recorder(err=_bad_request(f\"provider echoed your key {leak}\"))\n with pytest.raises(StructuredCallError) as ei:\n structured_call(EvidenceRef, [{\"role\": \"user\", \"content\": \"x\"}],\n client=_client(rec, api_key=leak), model=\"m\")\n exc = ei.value\n # from None:__cause__ 断开,__context__ 被抑制,exc_info 打印时不会带出原始异常\n assert exc.__cause__ is None\n assert exc.__suppress_context__ is True\n rendered = \"\".join(traceback.format_exception(type(exc), exc, exc.__traceback__))\n assert leak not in rendered, \"key 从异常链(exc_info 渲染)泄漏\"\n\n\n# ── 端到端:头 → 钳制 → ping 发的是真 run 的那个值 ───────────────────────────\n\n_BYOK = {\n \"X-LLM-Base-URL\": \"https://api.deepseek.com/v1\",\n \"X-LLM-API-Key\": \"sk-user-key\",\n \"X-LLM-Model\": \"deepseek-chat\",\n}\n\n\ndef test_ping_sends_capped_max_tokens_matching_real_run(db_path, monkeypatch):\n \"\"\"整条链的验收:声明上限 8192 → ping 实际发出 8192(= min(131072, 8192))。\n 这正是真 run 的 structured_call 会发的值 —— **连通性测试与真 run 逐字节一致**。\"\"\"\n rec = _Recorder()\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(rec))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n r = client.post(\"/llm/ping\", headers={**_BYOK, \"X-LLM-Max-Tokens\": \"8192\"})\n\n assert r.status_code == 200 and r.json()[\"ok\"] is True\n assert rec.calls[0][\"max_tokens\"] == 8192\n\n\ndef test_ping_without_cap_sends_full_default(db_path, monkeypatch):\n \"\"\"不声明上限(老配置 / 方舟)→ 发默认 131072,与改动前一致。\"\"\"\n rec = _Recorder()\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(rec))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n client.post(\"/llm/ping\", headers=_BYOK)\n\n assert rec.calls[0][\"max_tokens\"] == _DEFAULT_MAX_TOKENS\n\n\ndef test_ping_classifies_400_as_bad_request(db_path, monkeypatch):\n \"\"\"上限填得比厂商允许的大 → ping 当场分类 bad_request,前端渲染「按厂商文档调小」。\n 这就是「点一下按钮就知道」而不是「跑 7 分钟死 run 才知道」。\"\"\"\n rec = _Recorder(err=_bad_request())\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(rec))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n body = client.post(\"/llm/ping\", headers={**_BYOK, \"X-LLM-Max-Tokens\": \"999999\"}).json()\n\n assert body[\"ok\"] is False\n assert body[\"error_type\"] == \"bad_request\"\n assert \"max_tokens\" in body[\"detail\"]\n\n\n@pytest.mark.parametrize(\"bad\", [\"abc\", \"0\", \"-1\", \"1048577\", \"1.5\"])\ndef test_invalid_max_tokens_header_is_422_not_silently_ignored(db_path, monkeypatch, bad):\n \"\"\"非法值必须 422。静默忽略最坏 —— 用户以为设了上限,真 run 仍按 131072 发出去挨 400,\n 排查成本极高(界面显示「已配置 8192」,实际发的是 131072)。\"\"\"\n monkeypatch.setattr(\"rivalradar.api.deps.OpenAI\", lambda **kw: _client(_Recorder()))\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n\n r = client.post(\"/llm/ping\", headers={**_BYOK, \"X-LLM-Max-Tokens\": bad})\n\n assert r.status_code == 422\n assert \"X-LLM-Max-Tokens\" in r.json()[\"detail\"]\n\n\ndef test_max_tokens_header_alone_does_not_trigger_all_or_nothing(db_path):\n \"\"\"第 4 个头是**可选**的,不进「全有或全无」契约:只给它(无三头)仍应走 env fallback\n 路径的 503,而不是 422「BYOK 需三头齐全」—— 否则老前端一发这个头就全挂。\"\"\"\n client = TestClient(create_app(db_path=db_path, doubao_client=None))\n r = client.post(\"/llm/ping\", headers={\"X-LLM-Max-Tokens\": \"8192\"})\n assert r.status_code == 200\n assert r.json()[\"error_type\"] == \"unconfigured\"\n"}}}} |