1 line
1.1 MiB
Plaintext
1 line
1.1 MiB
Plaintext
{"repo": "alias8818/enoch-agentic-research-system", "files": {"tests/test_llm_settings.py": {"41dae2056e238e771a927f8675f80796783cec18": {"t": 1785746920.6638572, "n": 72292, "h": "3e96a6065d5d5438", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom datetime import datetime, timedelta, timezone\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane import read_models\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_model_health_summarizes_structured_output_capability_evidence() -> None:\n model_id = \"minimax/minimax-m3\"\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n\n class FakeStore:\n def event_page(self, **_kwargs: object):\n return (\n [\n {\n \"event_id\": 3,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"response_format_type\": \"json_schema\",\n \"checked_at\": \"2026-06-10T23:01:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 100,\n \"valid_json\": True,\n \"schema_ok\": True,\n \"malformed_kind\": \"\",\n },\n },\n {\n \"event_id\": 2,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": False,\n \"status_code\": 400,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n \"response_format_type\": \"json_object\",\n \"checked_at\": \"2026-06-10T23:00:00Z\",\n \"failure_kind\": \"unsupported_response_format\",\n \"error\": \"provider does not support response_format json_object\",\n },\n },\n {\n \"event_id\": 1,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"prompt_only\",\n \"response_format_type\": \"prompt_only\",\n \"checked_at\": \"2026-06-10T22:59:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 64,\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n },\n },\n ],\n None,\n False,\n )\n\n summary = read_models.llm_model_health_summary(FakeStore(), settings)\n\n capability = summary[\"structured_output_capabilities\"][f\"openrouter:{model_id}\"][\n \"candidate_json\"\n ]\n assert capability[\"schema_contract_name\"] == \"candidate_json/v1\"\n assert capability[\"recommended_response_format_type\"] == \"json_schema\"\n assert capability[\"modes\"][\"json_schema\"][\"status\"] == \"supported\"\n assert capability[\"modes\"][\"json_schema\"][\"schema_ok_rate\"] == 1\n assert capability[\"modes\"][\"json_object\"][\"status\"] == \"unsupported\"\n assert capability[\"modes\"][\"json_object\"][\"unsupported_mode_errors\"] == 1\n\n workflow = summary[\"workflow_recommendations\"][0]\n assert workflow[\"workflow_id\"] == \"research_generation\"\n assert workflow[\"status\"] == \"healthy\"\n assert workflow[\"route_policy\"][\"mode\"] == \"observe_only\"\n assert workflow[\"route_policy\"][\"production_route_mutation\"] is False\n assert workflow[\"route_policy\"][\"recommended_response_format_type\"] == \"\"\n assert workflow[\"route_policy\"][\"status\"] == \"blocked\"\n assert workflow[\"route_policy\"][\"promotion_gate\"][\"eligible\"] is False\n assert workflow[\"models\"][0][\"recommendation\"] == \"usable\"\n assert (\n workflow[\"models\"][0][\"contract_results\"][0][\"response_format_type\"]\n == \"json_schema\"\n )\n\n\ndef _iso_at(delta: timedelta) -> str:\n return (datetime.now(timezone.utc) + delta).replace(microsecond=0).isoformat()\n\n\ndef _settings_with_single_research_model(model_id: str = \"minimax/minimax-m3\"):\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n return settings\n\n\ndef _format_probe_event(\n event_id: int,\n *,\n model_id: str = \"minimax/minimax-m3\",\n mode: str = \"json_schema\",\n checked_at: str | None = None,\n valid_json: bool = True,\n schema_ok: bool = True,\n malformed_kind: str = \"\",\n finish_reason: str = \"stop\",\n visible_chars: int = 100,\n) -> dict[str, object]:\n return {\n \"event_id\": event_id,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": valid_json and schema_ok and not malformed_kind,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": mode,\n \"response_format_type\": mode,\n \"checked_at\": checked_at or _iso_at(timedelta(minutes=-5)),\n \"finish_reason\": finish_reason,\n \"visible_chars\": visible_chars,\n \"valid_json\": valid_json,\n \"schema_ok\": schema_ok,\n \"malformed_kind\": malformed_kind,\n },\n }\n\n\nclass _FakeLLMEventStore:\n def __init__(self, events: list[dict[str, object]]) -> None:\n self._events = events\n\n def event_page(self, **_kwargs: object):\n return (self._events, None, False)\n\n\ndef test_llm_route_policy_fails_closed_until_repeated_fresh_green_evidence() -> None:\n settings = _settings_with_single_research_model()\n summary = read_models.llm_model_health_summary(\n _FakeLLMEventStore([_format_probe_event(1)]), settings\n )\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"insufficient_green_attempts:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_fails_closed_on_stale_or_mixed_evidence() -> None:\n settings = _settings_with_single_research_model()\n stale = _iso_at(timedelta(days=-3))\n events = [\n _format_probe_event(4, checked_at=stale),\n _format_probe_event(3, checked_at=stale),\n _format_probe_event(2, checked_at=stale),\n _format_probe_event(\n 1,\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"stale_green_evidence:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_recommends_fail_closed_candidate_after_fresh_repeated_green_evidence() -> (\n None\n):\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(3, checked_at=_iso_at(timedelta(minutes=-1))),\n _format_probe_event(2, checked_at=_iso_at(timedelta(minutes=-2))),\n _format_probe_event(1, checked_at=_iso_at(timedelta(minutes=-3))),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"promotion_candidate\"\n assert policy[\"recommended_model\"] == \"minimax/minimax-m3\"\n assert policy[\"recommended_response_format_type\"] == \"json_schema\"\n assert policy[\"promotion_gate\"] == {\n \"eligible\": True,\n \"min_green_attempts\": 3,\n \"freshness_hours\": 24,\n \"reasons\": [],\n }\n\n\ndef test_llm_model_health_marks_mixed_provider_enforced_mode_unstable() -> None:\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(\n 3,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-1)),\n schema_ok=True,\n ),\n _format_probe_event(\n 2,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n _format_probe_event(\n 1,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-3)),\n schema_ok=True,\n ),\n ]\n\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n capability = summary[\"structured_output_capabilities\"][\n \"openrouter:minimax/minimax-m3\"\n ][\"candidate_json\"]\n mode = capability[\"modes\"][\"json_schema\"]\n assert mode[\"status\"] == \"unstable\"\n assert mode[\"success_count\"] == 2\n assert mode[\"failure_count\"] == 1\n assert capability[\"recommended_response_format_type\"] == \"\"\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_research_provider_api_key_surfaces_settings_load_failure(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n create_control_plane_router(config, lambda token: None)\n\n def fail_read_settings(_config: GateConfig) -> LLMSettings:\n raise ValueError(\"corrupt llm settings\")\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.read_llm_settings\",\n fail_read_settings,\n )\n\n with pytest.raises(\n RuntimeError, match=\"research provider settings could not be loaded\"\n ) as exc_info:\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n\n assert isinstance(exc_info.value.__cause__, ValueError)\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_format_probe_http_error_preserves_contract_and_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 400\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"unsupported response_format json_object\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"minimax/minimax-m3\",\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 400,\n \"Bad Request\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"minimax/minimax-m3\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is False\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"structured_output_mode\"] == \"json_object\"\n assert payload[\"response_format_type\"] == \"json_object\"\n assert payload[\"failure_kind\"] == \"unsupported_response_format\"\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_rejects_extra_candidates(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"One\\\\\",\\\\\"rationale\\\\\":\\\\\"A\\\\\"},'\n b'{\\\\\"title\\\\\":\\\\\"Two\\\\\",\\\\\"rationale\\\\\":\\\\\"B\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch=monkeypatch)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"valid_json\"] is True\n assert body[\"candidate_count\"] == 2\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"schema_mismatch\"\n\n\ndef test_llm_settings_openrouter_candidate_json_object_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"JSON object mode\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"candidate_count\"] == 1\n assert body[\"candidate_title_complete\"] is True\n assert body[\"candidate_rationale_complete\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"] == {\"type\": \"json_object\"}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"structured_output_mode\"] == \"json_object\"\n assert event_payload[\"response_format_type\"] == \"json_object\"\n assert event_payload[\"candidate_count\"] == 1\n assert event_payload[\"candidate_title_complete\"] is True\n assert event_payload[\"candidate_rationale_complete\"] is True\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "4afdcadbacff96c823fd52f244f10c63f830a34f": {"t": 1785769024.455482, "n": 72292, "h": "3e96a6065d5d5438", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom datetime import datetime, timedelta, timezone\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane import read_models\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_model_health_summarizes_structured_output_capability_evidence() -> None:\n model_id = \"minimax/minimax-m3\"\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n\n class FakeStore:\n def event_page(self, **_kwargs: object):\n return (\n [\n {\n \"event_id\": 3,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"response_format_type\": \"json_schema\",\n \"checked_at\": \"2026-06-10T23:01:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 100,\n \"valid_json\": True,\n \"schema_ok\": True,\n \"malformed_kind\": \"\",\n },\n },\n {\n \"event_id\": 2,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": False,\n \"status_code\": 400,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n \"response_format_type\": \"json_object\",\n \"checked_at\": \"2026-06-10T23:00:00Z\",\n \"failure_kind\": \"unsupported_response_format\",\n \"error\": \"provider does not support response_format json_object\",\n },\n },\n {\n \"event_id\": 1,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"prompt_only\",\n \"response_format_type\": \"prompt_only\",\n \"checked_at\": \"2026-06-10T22:59:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 64,\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n },\n },\n ],\n None,\n False,\n )\n\n summary = read_models.llm_model_health_summary(FakeStore(), settings)\n\n capability = summary[\"structured_output_capabilities\"][f\"openrouter:{model_id}\"][\n \"candidate_json\"\n ]\n assert capability[\"schema_contract_name\"] == \"candidate_json/v1\"\n assert capability[\"recommended_response_format_type\"] == \"json_schema\"\n assert capability[\"modes\"][\"json_schema\"][\"status\"] == \"supported\"\n assert capability[\"modes\"][\"json_schema\"][\"schema_ok_rate\"] == 1\n assert capability[\"modes\"][\"json_object\"][\"status\"] == \"unsupported\"\n assert capability[\"modes\"][\"json_object\"][\"unsupported_mode_errors\"] == 1\n\n workflow = summary[\"workflow_recommendations\"][0]\n assert workflow[\"workflow_id\"] == \"research_generation\"\n assert workflow[\"status\"] == \"healthy\"\n assert workflow[\"route_policy\"][\"mode\"] == \"observe_only\"\n assert workflow[\"route_policy\"][\"production_route_mutation\"] is False\n assert workflow[\"route_policy\"][\"recommended_response_format_type\"] == \"\"\n assert workflow[\"route_policy\"][\"status\"] == \"blocked\"\n assert workflow[\"route_policy\"][\"promotion_gate\"][\"eligible\"] is False\n assert workflow[\"models\"][0][\"recommendation\"] == \"usable\"\n assert (\n workflow[\"models\"][0][\"contract_results\"][0][\"response_format_type\"]\n == \"json_schema\"\n )\n\n\ndef _iso_at(delta: timedelta) -> str:\n return (datetime.now(timezone.utc) + delta).replace(microsecond=0).isoformat()\n\n\ndef _settings_with_single_research_model(model_id: str = \"minimax/minimax-m3\"):\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n return settings\n\n\ndef _format_probe_event(\n event_id: int,\n *,\n model_id: str = \"minimax/minimax-m3\",\n mode: str = \"json_schema\",\n checked_at: str | None = None,\n valid_json: bool = True,\n schema_ok: bool = True,\n malformed_kind: str = \"\",\n finish_reason: str = \"stop\",\n visible_chars: int = 100,\n) -> dict[str, object]:\n return {\n \"event_id\": event_id,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": valid_json and schema_ok and not malformed_kind,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": mode,\n \"response_format_type\": mode,\n \"checked_at\": checked_at or _iso_at(timedelta(minutes=-5)),\n \"finish_reason\": finish_reason,\n \"visible_chars\": visible_chars,\n \"valid_json\": valid_json,\n \"schema_ok\": schema_ok,\n \"malformed_kind\": malformed_kind,\n },\n }\n\n\nclass _FakeLLMEventStore:\n def __init__(self, events: list[dict[str, object]]) -> None:\n self._events = events\n\n def event_page(self, **_kwargs: object):\n return (self._events, None, False)\n\n\ndef test_llm_route_policy_fails_closed_until_repeated_fresh_green_evidence() -> None:\n settings = _settings_with_single_research_model()\n summary = read_models.llm_model_health_summary(\n _FakeLLMEventStore([_format_probe_event(1)]), settings\n )\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"insufficient_green_attempts:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_fails_closed_on_stale_or_mixed_evidence() -> None:\n settings = _settings_with_single_research_model()\n stale = _iso_at(timedelta(days=-3))\n events = [\n _format_probe_event(4, checked_at=stale),\n _format_probe_event(3, checked_at=stale),\n _format_probe_event(2, checked_at=stale),\n _format_probe_event(\n 1,\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"stale_green_evidence:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_recommends_fail_closed_candidate_after_fresh_repeated_green_evidence() -> (\n None\n):\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(3, checked_at=_iso_at(timedelta(minutes=-1))),\n _format_probe_event(2, checked_at=_iso_at(timedelta(minutes=-2))),\n _format_probe_event(1, checked_at=_iso_at(timedelta(minutes=-3))),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"promotion_candidate\"\n assert policy[\"recommended_model\"] == \"minimax/minimax-m3\"\n assert policy[\"recommended_response_format_type\"] == \"json_schema\"\n assert policy[\"promotion_gate\"] == {\n \"eligible\": True,\n \"min_green_attempts\": 3,\n \"freshness_hours\": 24,\n \"reasons\": [],\n }\n\n\ndef test_llm_model_health_marks_mixed_provider_enforced_mode_unstable() -> None:\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(\n 3,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-1)),\n schema_ok=True,\n ),\n _format_probe_event(\n 2,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n _format_probe_event(\n 1,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-3)),\n schema_ok=True,\n ),\n ]\n\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n capability = summary[\"structured_output_capabilities\"][\n \"openrouter:minimax/minimax-m3\"\n ][\"candidate_json\"]\n mode = capability[\"modes\"][\"json_schema\"]\n assert mode[\"status\"] == \"unstable\"\n assert mode[\"success_count\"] == 2\n assert mode[\"failure_count\"] == 1\n assert capability[\"recommended_response_format_type\"] == \"\"\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_research_provider_api_key_surfaces_settings_load_failure(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n create_control_plane_router(config, lambda token: None)\n\n def fail_read_settings(_config: GateConfig) -> LLMSettings:\n raise ValueError(\"corrupt llm settings\")\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.read_llm_settings\",\n fail_read_settings,\n )\n\n with pytest.raises(\n RuntimeError, match=\"research provider settings could not be loaded\"\n ) as exc_info:\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n\n assert isinstance(exc_info.value.__cause__, ValueError)\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_format_probe_http_error_preserves_contract_and_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 400\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"unsupported response_format json_object\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"minimax/minimax-m3\",\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 400,\n \"Bad Request\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"minimax/minimax-m3\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is False\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"structured_output_mode\"] == \"json_object\"\n assert payload[\"response_format_type\"] == \"json_object\"\n assert payload[\"failure_kind\"] == \"unsupported_response_format\"\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_rejects_extra_candidates(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"One\\\\\",\\\\\"rationale\\\\\":\\\\\"A\\\\\"},'\n b'{\\\\\"title\\\\\":\\\\\"Two\\\\\",\\\\\"rationale\\\\\":\\\\\"B\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch=monkeypatch)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"valid_json\"] is True\n assert body[\"candidate_count\"] == 2\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"schema_mismatch\"\n\n\ndef test_llm_settings_openrouter_candidate_json_object_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"JSON object mode\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"candidate_count\"] == 1\n assert body[\"candidate_title_complete\"] is True\n assert body[\"candidate_rationale_complete\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"] == {\"type\": \"json_object\"}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"structured_output_mode\"] == \"json_object\"\n assert event_payload[\"response_format_type\"] == \"json_object\"\n assert event_payload[\"candidate_count\"] == 1\n assert event_payload[\"candidate_title_complete\"] is True\n assert event_payload[\"candidate_rationale_complete\"] is True\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "855546c82e1a1173f93b2fe2867ff48596803d13": {"t": 1785769026.5012872, "n": 71493, "h": "3feafd407a2752e3", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom datetime import datetime, timedelta, timezone\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane import read_models\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_model_health_summarizes_structured_output_capability_evidence() -> None:\n model_id = \"minimax/minimax-m3\"\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n\n class FakeStore:\n def event_page(self, **_kwargs: object):\n return (\n [\n {\n \"event_id\": 3,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"response_format_type\": \"json_schema\",\n \"checked_at\": \"2026-06-10T23:01:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 100,\n \"valid_json\": True,\n \"schema_ok\": True,\n \"malformed_kind\": \"\",\n },\n },\n {\n \"event_id\": 2,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": False,\n \"status_code\": 400,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n \"response_format_type\": \"json_object\",\n \"checked_at\": \"2026-06-10T23:00:00Z\",\n \"failure_kind\": \"unsupported_response_format\",\n \"error\": \"provider does not support response_format json_object\",\n },\n },\n {\n \"event_id\": 1,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"prompt_only\",\n \"response_format_type\": \"prompt_only\",\n \"checked_at\": \"2026-06-10T22:59:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 64,\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n },\n },\n ],\n None,\n False,\n )\n\n summary = read_models.llm_model_health_summary(FakeStore(), settings)\n\n capability = summary[\"structured_output_capabilities\"][f\"openrouter:{model_id}\"][\n \"candidate_json\"\n ]\n assert capability[\"schema_contract_name\"] == \"candidate_json/v1\"\n assert capability[\"recommended_response_format_type\"] == \"json_schema\"\n assert capability[\"modes\"][\"json_schema\"][\"status\"] == \"supported\"\n assert capability[\"modes\"][\"json_schema\"][\"schema_ok_rate\"] == 1\n assert capability[\"modes\"][\"json_object\"][\"status\"] == \"unsupported\"\n assert capability[\"modes\"][\"json_object\"][\"unsupported_mode_errors\"] == 1\n\n workflow = summary[\"workflow_recommendations\"][0]\n assert workflow[\"workflow_id\"] == \"research_generation\"\n assert workflow[\"status\"] == \"healthy\"\n assert workflow[\"route_policy\"][\"mode\"] == \"observe_only\"\n assert workflow[\"route_policy\"][\"production_route_mutation\"] is False\n assert workflow[\"route_policy\"][\"recommended_response_format_type\"] == \"\"\n assert workflow[\"route_policy\"][\"status\"] == \"blocked\"\n assert workflow[\"route_policy\"][\"promotion_gate\"][\"eligible\"] is False\n assert workflow[\"models\"][0][\"recommendation\"] == \"usable\"\n assert (\n workflow[\"models\"][0][\"contract_results\"][0][\"response_format_type\"]\n == \"json_schema\"\n )\n\n\ndef _iso_at(delta: timedelta) -> str:\n return (datetime.now(timezone.utc) + delta).replace(microsecond=0).isoformat()\n\n\ndef _settings_with_single_research_model(model_id: str = \"minimax/minimax-m3\"):\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n return settings\n\n\ndef _format_probe_event(\n event_id: int,\n *,\n model_id: str = \"minimax/minimax-m3\",\n mode: str = \"json_schema\",\n checked_at: str | None = None,\n valid_json: bool = True,\n schema_ok: bool = True,\n malformed_kind: str = \"\",\n finish_reason: str = \"stop\",\n visible_chars: int = 100,\n) -> dict[str, object]:\n return {\n \"event_id\": event_id,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": valid_json and schema_ok and not malformed_kind,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": mode,\n \"response_format_type\": mode,\n \"checked_at\": checked_at or _iso_at(timedelta(minutes=-5)),\n \"finish_reason\": finish_reason,\n \"visible_chars\": visible_chars,\n \"valid_json\": valid_json,\n \"schema_ok\": schema_ok,\n \"malformed_kind\": malformed_kind,\n },\n }\n\n\nclass _FakeLLMEventStore:\n def __init__(self, events: list[dict[str, object]]) -> None:\n self._events = events\n\n def event_page(self, **_kwargs: object):\n return (self._events, None, False)\n\n\ndef test_llm_route_policy_fails_closed_until_repeated_fresh_green_evidence() -> None:\n settings = _settings_with_single_research_model()\n summary = read_models.llm_model_health_summary(\n _FakeLLMEventStore([_format_probe_event(1)]), settings\n )\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"insufficient_green_attempts:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_fails_closed_on_stale_or_mixed_evidence() -> None:\n settings = _settings_with_single_research_model()\n stale = _iso_at(timedelta(days=-3))\n events = [\n _format_probe_event(4, checked_at=stale),\n _format_probe_event(3, checked_at=stale),\n _format_probe_event(2, checked_at=stale),\n _format_probe_event(\n 1,\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"stale_green_evidence:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_recommends_fail_closed_candidate_after_fresh_repeated_green_evidence() -> (\n None\n):\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(3, checked_at=_iso_at(timedelta(minutes=-1))),\n _format_probe_event(2, checked_at=_iso_at(timedelta(minutes=-2))),\n _format_probe_event(1, checked_at=_iso_at(timedelta(minutes=-3))),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"promotion_candidate\"\n assert policy[\"recommended_model\"] == \"minimax/minimax-m3\"\n assert policy[\"recommended_response_format_type\"] == \"json_schema\"\n assert policy[\"promotion_gate\"] == {\n \"eligible\": True,\n \"min_green_attempts\": 3,\n \"freshness_hours\": 24,\n \"reasons\": [],\n }\n\n\ndef test_llm_model_health_marks_mixed_provider_enforced_mode_unstable() -> None:\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(\n 3,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-1)),\n schema_ok=True,\n ),\n _format_probe_event(\n 2,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n _format_probe_event(\n 1,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-3)),\n schema_ok=True,\n ),\n ]\n\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n capability = summary[\"structured_output_capabilities\"][\n \"openrouter:minimax/minimax-m3\"\n ][\"candidate_json\"]\n mode = capability[\"modes\"][\"json_schema\"]\n assert mode[\"status\"] == \"unstable\"\n assert mode[\"success_count\"] == 2\n assert mode[\"failure_count\"] == 1\n assert capability[\"recommended_response_format_type\"] == \"\"\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_format_probe_http_error_preserves_contract_and_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 400\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"unsupported response_format json_object\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"minimax/minimax-m3\",\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 400,\n \"Bad Request\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"minimax/minimax-m3\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is False\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"structured_output_mode\"] == \"json_object\"\n assert payload[\"response_format_type\"] == \"json_object\"\n assert payload[\"failure_kind\"] == \"unsupported_response_format\"\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_rejects_extra_candidates(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"One\\\\\",\\\\\"rationale\\\\\":\\\\\"A\\\\\"},'\n b'{\\\\\"title\\\\\":\\\\\"Two\\\\\",\\\\\"rationale\\\\\":\\\\\"B\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch=monkeypatch)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"valid_json\"] is True\n assert body[\"candidate_count\"] == 2\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"schema_mismatch\"\n\n\ndef test_llm_settings_openrouter_candidate_json_object_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"JSON object mode\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"candidate_count\"] == 1\n assert body[\"candidate_title_complete\"] is True\n assert body[\"candidate_rationale_complete\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"] == {\"type\": \"json_object\"}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"structured_output_mode\"] == \"json_object\"\n assert event_payload[\"response_format_type\"] == \"json_object\"\n assert event_payload[\"candidate_count\"] == 1\n assert event_payload[\"candidate_title_complete\"] is True\n assert event_payload[\"candidate_rationale_complete\"] is True\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int, **_kwargs): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urlopen_validated\",\n lambda _req, timeout, **_kwargs: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "6b60f886a3b55de15bfc5c0a82a0e281e33649e5": {"t": 1785769030.3432097, "n": 71433, "h": "a2d749f915036950", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom datetime import datetime, timedelta, timezone\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane import read_models\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_model_health_summarizes_structured_output_capability_evidence() -> None:\n model_id = \"minimax/minimax-m3\"\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n\n class FakeStore:\n def event_page(self, **_kwargs: object):\n return (\n [\n {\n \"event_id\": 3,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"response_format_type\": \"json_schema\",\n \"checked_at\": \"2026-06-10T23:01:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 100,\n \"valid_json\": True,\n \"schema_ok\": True,\n \"malformed_kind\": \"\",\n },\n },\n {\n \"event_id\": 2,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": False,\n \"status_code\": 400,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n \"response_format_type\": \"json_object\",\n \"checked_at\": \"2026-06-10T23:00:00Z\",\n \"failure_kind\": \"unsupported_response_format\",\n \"error\": \"provider does not support response_format json_object\",\n },\n },\n {\n \"event_id\": 1,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"prompt_only\",\n \"response_format_type\": \"prompt_only\",\n \"checked_at\": \"2026-06-10T22:59:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 64,\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n },\n },\n ],\n None,\n False,\n )\n\n summary = read_models.llm_model_health_summary(FakeStore(), settings)\n\n capability = summary[\"structured_output_capabilities\"][f\"openrouter:{model_id}\"][\n \"candidate_json\"\n ]\n assert capability[\"schema_contract_name\"] == \"candidate_json/v1\"\n assert capability[\"recommended_response_format_type\"] == \"json_schema\"\n assert capability[\"modes\"][\"json_schema\"][\"status\"] == \"supported\"\n assert capability[\"modes\"][\"json_schema\"][\"schema_ok_rate\"] == 1\n assert capability[\"modes\"][\"json_object\"][\"status\"] == \"unsupported\"\n assert capability[\"modes\"][\"json_object\"][\"unsupported_mode_errors\"] == 1\n\n workflow = summary[\"workflow_recommendations\"][0]\n assert workflow[\"workflow_id\"] == \"research_generation\"\n assert workflow[\"status\"] == \"healthy\"\n assert workflow[\"route_policy\"][\"mode\"] == \"observe_only\"\n assert workflow[\"route_policy\"][\"production_route_mutation\"] is False\n assert workflow[\"route_policy\"][\"recommended_response_format_type\"] == \"\"\n assert workflow[\"route_policy\"][\"status\"] == \"blocked\"\n assert workflow[\"route_policy\"][\"promotion_gate\"][\"eligible\"] is False\n assert workflow[\"models\"][0][\"recommendation\"] == \"usable\"\n assert (\n workflow[\"models\"][0][\"contract_results\"][0][\"response_format_type\"]\n == \"json_schema\"\n )\n\n\ndef _iso_at(delta: timedelta) -> str:\n return (datetime.now(timezone.utc) + delta).replace(microsecond=0).isoformat()\n\n\ndef _settings_with_single_research_model(model_id: str = \"minimax/minimax-m3\"):\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n return settings\n\n\ndef _format_probe_event(\n event_id: int,\n *,\n model_id: str = \"minimax/minimax-m3\",\n mode: str = \"json_schema\",\n checked_at: str | None = None,\n valid_json: bool = True,\n schema_ok: bool = True,\n malformed_kind: str = \"\",\n finish_reason: str = \"stop\",\n visible_chars: int = 100,\n) -> dict[str, object]:\n return {\n \"event_id\": event_id,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": valid_json and schema_ok and not malformed_kind,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": mode,\n \"response_format_type\": mode,\n \"checked_at\": checked_at or _iso_at(timedelta(minutes=-5)),\n \"finish_reason\": finish_reason,\n \"visible_chars\": visible_chars,\n \"valid_json\": valid_json,\n \"schema_ok\": schema_ok,\n \"malformed_kind\": malformed_kind,\n },\n }\n\n\nclass _FakeLLMEventStore:\n def __init__(self, events: list[dict[str, object]]) -> None:\n self._events = events\n\n def event_page(self, **_kwargs: object):\n return (self._events, None, False)\n\n\ndef test_llm_route_policy_fails_closed_until_repeated_fresh_green_evidence() -> None:\n settings = _settings_with_single_research_model()\n summary = read_models.llm_model_health_summary(\n _FakeLLMEventStore([_format_probe_event(1)]), settings\n )\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"insufficient_green_attempts:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_fails_closed_on_stale_or_mixed_evidence() -> None:\n settings = _settings_with_single_research_model()\n stale = _iso_at(timedelta(days=-3))\n events = [\n _format_probe_event(4, checked_at=stale),\n _format_probe_event(3, checked_at=stale),\n _format_probe_event(2, checked_at=stale),\n _format_probe_event(\n 1,\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"stale_green_evidence:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_recommends_fail_closed_candidate_after_fresh_repeated_green_evidence() -> (\n None\n):\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(3, checked_at=_iso_at(timedelta(minutes=-1))),\n _format_probe_event(2, checked_at=_iso_at(timedelta(minutes=-2))),\n _format_probe_event(1, checked_at=_iso_at(timedelta(minutes=-3))),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"promotion_candidate\"\n assert policy[\"recommended_model\"] == \"minimax/minimax-m3\"\n assert policy[\"recommended_response_format_type\"] == \"json_schema\"\n assert policy[\"promotion_gate\"] == {\n \"eligible\": True,\n \"min_green_attempts\": 3,\n \"freshness_hours\": 24,\n \"reasons\": [],\n }\n\n\ndef test_llm_model_health_marks_mixed_provider_enforced_mode_unstable() -> None:\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(\n 3,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-1)),\n schema_ok=True,\n ),\n _format_probe_event(\n 2,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n _format_probe_event(\n 1,\n mode=\"json_schema\",\n checked_at=_iso_at(timedelta(minutes=-3)),\n schema_ok=True,\n ),\n ]\n\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n capability = summary[\"structured_output_capabilities\"][\n \"openrouter:minimax/minimax-m3\"\n ][\"candidate_json\"]\n mode = capability[\"modes\"][\"json_schema\"]\n assert mode[\"status\"] == \"unstable\"\n assert mode[\"success_count\"] == 2\n assert mode[\"failure_count\"] == 1\n assert capability[\"recommended_response_format_type\"] == \"\"\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_format_probe_http_error_preserves_contract_and_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 400\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"unsupported response_format json_object\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"minimax/minimax-m3\",\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 400,\n \"Bad Request\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"minimax/minimax-m3\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is False\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"structured_output_mode\"] == \"json_object\"\n assert payload[\"response_format_type\"] == \"json_object\"\n assert payload[\"failure_kind\"] == \"unsupported_response_format\"\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_rejects_extra_candidates(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"One\\\\\",\\\\\"rationale\\\\\":\\\\\"A\\\\\"},'\n b'{\\\\\"title\\\\\":\\\\\"Two\\\\\",\\\\\"rationale\\\\\":\\\\\"B\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch=monkeypatch)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"valid_json\"] is True\n assert body[\"candidate_count\"] == 2\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"schema_mismatch\"\n\n\ndef test_llm_settings_openrouter_candidate_json_object_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"JSON object mode\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"candidate_count\"] == 1\n assert body[\"candidate_title_complete\"] is True\n assert body[\"candidate_rationale_complete\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"] == {\"type\": \"json_object\"}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"structured_output_mode\"] == \"json_object\"\n assert event_payload[\"response_format_type\"] == \"json_object\"\n assert event_payload[\"candidate_count\"] == 1\n assert event_payload[\"candidate_title_complete\"] is True\n assert event_payload[\"candidate_rationale_complete\"] is True\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "267e9b274f25d6d5b2645d2ae56783bfecc9bb19": {"t": 1785769031.8411224, "n": 69979, "h": "a0cffc7180584dcb", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom datetime import datetime, timedelta, timezone\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane import read_models\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_model_health_summarizes_structured_output_capability_evidence() -> None:\n model_id = \"minimax/minimax-m3\"\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n\n class FakeStore:\n def event_page(self, **_kwargs: object):\n return (\n [\n {\n \"event_id\": 3,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"response_format_type\": \"json_schema\",\n \"checked_at\": \"2026-06-10T23:01:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 100,\n \"valid_json\": True,\n \"schema_ok\": True,\n \"malformed_kind\": \"\",\n },\n },\n {\n \"event_id\": 2,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": False,\n \"status_code\": 400,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n \"response_format_type\": \"json_object\",\n \"checked_at\": \"2026-06-10T23:00:00Z\",\n \"failure_kind\": \"unsupported_response_format\",\n \"error\": \"provider does not support response_format json_object\",\n },\n },\n {\n \"event_id\": 1,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"prompt_only\",\n \"response_format_type\": \"prompt_only\",\n \"checked_at\": \"2026-06-10T22:59:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 64,\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n },\n },\n ],\n None,\n False,\n )\n\n summary = read_models.llm_model_health_summary(FakeStore(), settings)\n\n capability = summary[\"structured_output_capabilities\"][f\"openrouter:{model_id}\"][\n \"candidate_json\"\n ]\n assert capability[\"schema_contract_name\"] == \"candidate_json/v1\"\n assert capability[\"recommended_response_format_type\"] == \"json_schema\"\n assert capability[\"modes\"][\"json_schema\"][\"status\"] == \"supported\"\n assert capability[\"modes\"][\"json_schema\"][\"schema_ok_rate\"] == 1\n assert capability[\"modes\"][\"json_object\"][\"status\"] == \"unsupported\"\n assert capability[\"modes\"][\"json_object\"][\"unsupported_mode_errors\"] == 1\n\n workflow = summary[\"workflow_recommendations\"][0]\n assert workflow[\"workflow_id\"] == \"research_generation\"\n assert workflow[\"status\"] == \"healthy\"\n assert workflow[\"route_policy\"][\"mode\"] == \"observe_only\"\n assert workflow[\"route_policy\"][\"production_route_mutation\"] is False\n assert workflow[\"route_policy\"][\"recommended_response_format_type\"] == \"\"\n assert workflow[\"route_policy\"][\"status\"] == \"blocked\"\n assert workflow[\"route_policy\"][\"promotion_gate\"][\"eligible\"] is False\n assert workflow[\"models\"][0][\"recommendation\"] == \"usable\"\n assert (\n workflow[\"models\"][0][\"contract_results\"][0][\"response_format_type\"]\n == \"json_schema\"\n )\n\n\ndef _iso_at(delta: timedelta) -> str:\n return (datetime.now(timezone.utc) + delta).replace(microsecond=0).isoformat()\n\n\ndef _settings_with_single_research_model(model_id: str = \"minimax/minimax-m3\"):\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n return settings\n\n\ndef _format_probe_event(\n event_id: int,\n *,\n model_id: str = \"minimax/minimax-m3\",\n mode: str = \"json_schema\",\n checked_at: str | None = None,\n valid_json: bool = True,\n schema_ok: bool = True,\n malformed_kind: str = \"\",\n finish_reason: str = \"stop\",\n visible_chars: int = 100,\n) -> dict[str, object]:\n return {\n \"event_id\": event_id,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": valid_json and schema_ok and not malformed_kind,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": mode,\n \"response_format_type\": mode,\n \"checked_at\": checked_at or _iso_at(timedelta(minutes=-5)),\n \"finish_reason\": finish_reason,\n \"visible_chars\": visible_chars,\n \"valid_json\": valid_json,\n \"schema_ok\": schema_ok,\n \"malformed_kind\": malformed_kind,\n },\n }\n\n\nclass _FakeLLMEventStore:\n def __init__(self, events: list[dict[str, object]]) -> None:\n self._events = events\n\n def event_page(self, **_kwargs: object):\n return (self._events, None, False)\n\n\ndef test_llm_route_policy_fails_closed_until_repeated_fresh_green_evidence() -> None:\n settings = _settings_with_single_research_model()\n summary = read_models.llm_model_health_summary(\n _FakeLLMEventStore([_format_probe_event(1)]), settings\n )\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"insufficient_green_attempts:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_fails_closed_on_stale_or_mixed_evidence() -> None:\n settings = _settings_with_single_research_model()\n stale = _iso_at(timedelta(days=-3))\n events = [\n _format_probe_event(4, checked_at=stale),\n _format_probe_event(3, checked_at=stale),\n _format_probe_event(2, checked_at=stale),\n _format_probe_event(\n 1,\n checked_at=_iso_at(timedelta(minutes=-2)),\n schema_ok=False,\n malformed_kind=\"schema_mismatch\",\n ),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"status\"] == \"blocked\"\n assert policy[\"recommended_model\"] == \"\"\n assert policy[\"recommended_response_format_type\"] == \"\"\n assert policy[\"promotion_gate\"][\"eligible\"] is False\n assert (\n \"stale_green_evidence:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n assert (\n \"non_green_recent_attempt:candidate_json:json_schema\"\n in policy[\"promotion_gate\"][\"reasons\"]\n )\n\n\ndef test_llm_route_policy_recommends_fail_closed_candidate_after_fresh_repeated_green_evidence() -> (\n None\n):\n settings = _settings_with_single_research_model()\n events = [\n _format_probe_event(3, checked_at=_iso_at(timedelta(minutes=-1))),\n _format_probe_event(2, checked_at=_iso_at(timedelta(minutes=-2))),\n _format_probe_event(1, checked_at=_iso_at(timedelta(minutes=-3))),\n ]\n summary = read_models.llm_model_health_summary(_FakeLLMEventStore(events), settings)\n\n policy = summary[\"workflow_recommendations\"][0][\"route_policy\"]\n\n assert policy[\"mode\"] == \"observe_only\"\n assert policy[\"production_route_mutation\"] is False\n assert policy[\"status\"] == \"promotion_candidate\"\n assert policy[\"recommended_model\"] == \"minimax/minimax-m3\"\n assert policy[\"recommended_response_format_type\"] == \"json_schema\"\n assert policy[\"promotion_gate\"] == {\n \"eligible\": True,\n \"min_green_attempts\": 3,\n \"freshness_hours\": 24,\n \"reasons\": [],\n }\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_format_probe_http_error_preserves_contract_and_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 400\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"unsupported response_format json_object\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"minimax/minimax-m3\",\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 400,\n \"Bad Request\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"minimax/minimax-m3\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is False\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"structured_output_mode\"] == \"json_object\"\n assert payload[\"response_format_type\"] == \"json_object\"\n assert payload[\"failure_kind\"] == \"unsupported_response_format\"\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_rejects_extra_candidates(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"One\\\\\",\\\\\"rationale\\\\\":\\\\\"A\\\\\"},'\n b'{\\\\\"title\\\\\":\\\\\"Two\\\\\",\\\\\"rationale\\\\\":\\\\\"B\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch=monkeypatch)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"valid_json\"] is True\n assert body[\"candidate_count\"] == 2\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"schema_mismatch\"\n\n\ndef test_llm_settings_openrouter_candidate_json_object_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"JSON object mode\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"candidate_count\"] == 1\n assert body[\"candidate_title_complete\"] is True\n assert body[\"candidate_rationale_complete\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"] == {\"type\": \"json_object\"}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"structured_output_mode\"] == \"json_object\"\n assert event_payload[\"response_format_type\"] == \"json_object\"\n assert event_payload[\"candidate_count\"] == 1\n assert event_payload[\"candidate_title_complete\"] is True\n assert event_payload[\"candidate_rationale_complete\"] is True\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "5b0ea8c6fdd4a9ff6b8620473749ee2de4a3c57d": {"t": 1785769034.0941112, "n": 64685, "h": "0b822e009dc0d8a0", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane import read_models\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_model_health_summarizes_structured_output_capability_evidence() -> None:\n model_id = \"minimax/minimax-m3\"\n settings = default_llm_settings()\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=model_id,\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n research_generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research_generation.provider_ids = [\"openrouter\"]\n research_generation.model_pool = [model_id]\n research_generation.default_model = model_id\n\n class FakeStore:\n def event_page(self, **_kwargs: object):\n return (\n [\n {\n \"event_id\": 3,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"response_format_type\": \"json_schema\",\n \"checked_at\": \"2026-06-10T23:01:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 100,\n \"valid_json\": True,\n \"schema_ok\": True,\n \"malformed_kind\": \"\",\n },\n },\n {\n \"event_id\": 2,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": False,\n \"status_code\": 400,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n \"response_format_type\": \"json_object\",\n \"checked_at\": \"2026-06-10T23:00:00Z\",\n \"failure_kind\": \"unsupported_response_format\",\n \"error\": \"provider does not support response_format json_object\",\n },\n },\n {\n \"event_id\": 1,\n \"payload\": {\n \"provider_id\": \"openrouter\",\n \"model_id\": model_id,\n \"ok\": True,\n \"status_code\": 200,\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"prompt_only\",\n \"response_format_type\": \"prompt_only\",\n \"checked_at\": \"2026-06-10T22:59:00Z\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 64,\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n },\n },\n ],\n None,\n False,\n )\n\n summary = read_models.llm_model_health_summary(FakeStore(), settings)\n\n capability = summary[\"structured_output_capabilities\"][f\"openrouter:{model_id}\"][\n \"candidate_json\"\n ]\n assert capability[\"schema_contract_name\"] == \"candidate_json/v1\"\n assert capability[\"recommended_response_format_type\"] == \"json_schema\"\n assert capability[\"modes\"][\"json_schema\"][\"status\"] == \"supported\"\n assert capability[\"modes\"][\"json_schema\"][\"schema_ok_rate\"] == 1\n assert capability[\"modes\"][\"json_object\"][\"status\"] == \"unsupported\"\n assert capability[\"modes\"][\"json_object\"][\"unsupported_mode_errors\"] == 1\n\n workflow = summary[\"workflow_recommendations\"][0]\n assert workflow[\"workflow_id\"] == \"research_generation\"\n assert workflow[\"status\"] == \"healthy\"\n assert workflow[\"route_policy\"][\"mode\"] == \"observe_only\"\n assert workflow[\"route_policy\"][\"production_route_mutation\"] is False\n assert workflow[\"route_policy\"][\"recommended_response_format_type\"] == \"json_schema\"\n assert workflow[\"models\"][0][\"recommendation\"] == \"usable\"\n assert (\n workflow[\"models\"][0][\"contract_results\"][0][\"response_format_type\"]\n == \"json_schema\"\n )\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_format_probe_http_error_preserves_contract_and_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 400\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"unsupported response_format json_object\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"minimax/minimax-m3\",\n provider_id=\"openrouter\",\n label=\"MiniMax M3\",\n enabled=True,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 400,\n \"Bad Request\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"minimax/minimax-m3\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is False\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"structured_output_mode\"] == \"json_object\"\n assert payload[\"response_format_type\"] == \"json_object\"\n assert payload[\"failure_kind\"] == \"unsupported_response_format\"\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_rejects_extra_candidates(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"One\\\\\",\\\\\"rationale\\\\\":\\\\\"A\\\\\"},'\n b'{\\\\\"title\\\\\":\\\\\"Two\\\\\",\\\\\"rationale\\\\\":\\\\\"B\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch=monkeypatch)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"valid_json\"] is True\n assert body[\"candidate_count\"] == 2\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"schema_mismatch\"\n\n\ndef test_llm_settings_openrouter_candidate_json_object_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"JSON object mode\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"candidate_count\"] == 1\n assert body[\"candidate_title_complete\"] is True\n assert body[\"candidate_rationale_complete\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"] == {\"type\": \"json_object\"}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"structured_output_mode\"] == \"json_object\"\n assert event_payload[\"response_format_type\"] == \"json_object\"\n assert event_payload[\"candidate_count\"] == 1\n assert event_payload[\"candidate_title_complete\"] is True\n assert event_payload[\"candidate_rationale_complete\"] is True\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "bf3258b95214d6167b00d9fd389564dc6d5d734b": {"t": 1785769035.7774167, "n": 56785, "h": "259f395781a7c3fb", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_rejects_extra_candidates(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"One\\\\\",\\\\\"rationale\\\\\":\\\\\"A\\\\\"},'\n b'{\\\\\"title\\\\\":\\\\\"Two\\\\\",\\\\\"rationale\\\\\":\\\\\"B\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch=monkeypatch)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"valid_json\"] is True\n assert body[\"candidate_count\"] == 2\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"schema_mismatch\"\n\n\ndef test_llm_settings_openrouter_candidate_json_object_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"JSON object mode\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_object\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_object\"\n assert body[\"response_format_type\"] == \"json_object\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"candidate_count\"] == 1\n assert body[\"candidate_title_complete\"] is True\n assert body[\"candidate_rationale_complete\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"] == {\"type\": \"json_object\"}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"structured_output_mode\"] == \"json_object\"\n assert event_payload[\"response_format_type\"] == \"json_object\"\n assert event_payload[\"candidate_count\"] == 1\n assert event_payload[\"candidate_title_complete\"] is True\n assert event_payload[\"candidate_rationale_complete\"] is True\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "a9eab35355aa07479979e42bf423fb02e2ab2511": {"t": 1785769042.357559, "n": 50923, "h": "5e4fbcfd60b826b1", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_llm_model_health_summary_surfaces_recoverable_candidate_json_shape() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:legacy-array\",\n provider_id=\"synthetic\",\n label=\"Legacy Array\",\n enabled=True,\n )\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:legacy-array\"]\n generation.default_model = \"hf:legacy-array\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-10T02:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:legacy-array\",\n \"ok\": True,\n \"checked_at\": \"2026-06-10T02:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": False,\n \"malformed_kind\": \"legacy_candidate_array_shape\",\n \"recoverable_json_shape\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n }\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n row = summary[\"models\"][0]\n assert row[\"endpoint_health\"] == \"healthy\"\n assert row[\"format_health\"] == \"recoverable_mismatch\"\n assert row[\"recoverable_json_shape_count\"] == 1\n assert row[\"latest_recoverable_json_shape\"] is True\n assert \"prompt/schema parser recovery\" in row[\"operator_action\"]\n assert summary[\"structurally_unhealthy_count\"] == 1\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n model_rec = generation_rec[\"models\"][0]\n assert model_rec[\"recommendation\"] == \"repair_recoverable_shape\"\n assert model_rec[\"recoverable_shape_failures\"] == [\"candidate_json\"]\n assert model_rec[\"contract_results\"][0][\"recoverable_json_shape\"] is True\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "f50e679320da0d54944fa749266dd9cfa23af363": {"t": 1785769044.241395, "n": 48178, "h": "dc1ef2314db6026b", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> (\n None\n):\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "a61d789d81d64eb3b2ff23d633289da3ef09128c": {"t": 1785769046.2314692, "n": 48170, "h": "1201d54bfaf48714", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_array_shape_is_recoverable_mismatch() -> None:\n from enoch_control_plane.control_plane.router import _evaluate_llm_format_probe\n\n result = _evaluate_llm_format_probe(\n \"candidate_json\",\n visible_text='[{\"title\":\"Probe\",\"rationale\":\"Legacy array shape\"}]',\n finish_reason=\"stop\",\n )\n\n assert result[\"valid_json\"] is True\n assert result[\"schema_ok\"] is False\n assert result[\"malformed_kind\"] == \"legacy_candidate_array_shape\"\n assert result[\"recoverable_json_shape\"] is True\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "d60a9cfc0e1f56c009c91d8f0750ac3cf548d366": {"t": 1785769048.1469634, "n": 47610, "h": "bace74afcfd65a24", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_prompt_matches_schema_shape() -> None:\n from enoch_control_plane.control_plane.router import (\n _llm_format_probe_prompt,\n _llm_probe_json_schema,\n )\n\n prompt = _llm_format_probe_prompt(\"candidate_json\")\n schema = _llm_probe_json_schema(\"candidate_json\")\n\n assert schema[\"type\"] == \"object\"\n assert schema[\"required\"] == [\"candidates\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n assert \"compact JSON array\" not in prompt\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"{\\\\\"candidates\\\\\":[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",'\n b'\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n request_payload = json.loads(str(seen[\"body\"]))\n prompt = request_payload[\"messages\"][0][\"content\"]\n assert \"compact JSON object\" in prompt\n assert '\"candidates\"' in prompt\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "34a9518bde2951ee2cee8bf09faf620ac1bc04dd": {"t": 1785769050.148198, "n": 46888, "h": "0ac021e2d4327069", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_update_event_error_is_generic(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FailingEventStore:\n def append_event(self, **_kwargs: object) -> int:\n raise RuntimeError(\"stack trace details must stay server-side\")\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config, monkeypatch, FailingEventStore())\n settings = default_llm_settings(config)\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"event_error\"] == \"settings update event could not be recorded\"\n assert \"RuntimeError\" not in response.text\n assert \"stack trace details\" not in response.text\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n assert \"compact JSON array\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "ca73cc41f6c90302577c6bcf5a42026f130664aa": {"t": 1785769052.1650553, "n": 45930, "h": "f8d8b825cc028d7a", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Enough budget\\\\\"}]\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n assert \"compact JSON array\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\"title\\\\\":\\\\\"Probe\\\\\",\\\\\"rationale\\\\\":\\\\\"Schema\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "a9309267f02caf89c058a3f89997f27187c86e1a": {"t": 1785769054.9338634, "n": 45946, "h": "d12e54f6d6cc9a9b", "c": "from __future__ import annotations\n\nimport json\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"[{\\\\\\\"title\\\\\\\":\\\\\\\"Probe\\\\\\\",\\\\\\\"rationale\\\\\\\":\\\\\\\"Enough budget\\\\\\\"}]\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n assert \"compact JSON array\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_openrouter_candidate_schema_probe_records_structured_mode(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"candidates\\\\\":'\n b'[{\\\\\\\"title\\\\\\\":\\\\\\\"Probe\\\\\\\",\\\\\\\"rationale\\\\\\\":\\\\\\\"Schema\\\\\\\"}]}\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"deepseek/deepseek-v4-pro\",\n provider_id=\"openrouter\",\n label=\"DeepSeek V4 Pro\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"deepseek/deepseek-v4-pro\",\n \"source\": \"manual\",\n \"prompt_contract\": \"candidate_json\",\n \"structured_output_mode\": \"json_schema\",\n \"reasoning_effort\": \"low\",\n \"reasoning_exclude\": True,\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"structured_output_mode\"] == \"json_schema\"\n assert body[\"response_format_type\"] == \"json_schema\"\n assert body[\"reasoning_effort\"] == \"low\"\n assert body[\"reasoning_excluded\"] is True\n\n request_payload = json.loads(str(seen[\"body\"]))\n assert request_payload[\"response_format\"][\"type\"] == \"json_schema\"\n schema = request_payload[\"response_format\"][\"json_schema\"]\n assert schema[\"name\"] == \"enoch_candidate_probe\"\n assert schema[\"strict\"] is True\n assert schema[\"schema\"][\"required\"] == [\"candidates\"]\n assert request_payload[\"reasoning\"] == {\"effort\": \"low\", \"exclude\": True}\n\n event_payload = store.events[0][\"payload\"]\n assert event_payload[\"source\"] == \"format_probe\"\n assert event_payload[\"structured_output_mode\"] == \"json_schema\"\n assert event_payload[\"response_format_type\"] == \"json_schema\"\n assert event_payload[\"reasoning_effort\"] == \"low\"\n assert event_payload[\"reasoning_excluded\"] is True\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "50d5b7dffb0ac5a25ebc4620e523b5dfeadd663c": {"t": 1785769056.8847015, "n": 41892, "h": "eb697db1ffe76fa9", "c": "from __future__ import annotations\n\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"max_tokens\"] == 128\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_candidate_json_probe_uses_structured_output_budget(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":'\n b'\"[{\\\\\\\"title\\\\\\\":\\\\\\\"Probe\\\\\\\",\\\\\\\"rationale\\\\\\\":\\\\\\\"Enough budget\\\\\\\"}]\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/candidate-json\",\n provider_id=\"openrouter\",\n label=\"Owl Candidate JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/candidate-json\",\n \"prompt_contract\": \"candidate_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"candidate_json\"\n assert body[\"max_tokens\"] == 1024\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 1024' in str(seen[\"body\"])\n assert \"compact JSON array\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"candidate_json\"\n assert payload[\"max_tokens\"] == 1024\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "56fb42849c64f4840f16ac11516b8149a3e6e4d0": {"t": 1785769058.4738178, "n": 38547, "h": "d01d19e9b0d1f120", "c": "from __future__ import annotations\n\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_llm_model_health_summary_recommends_workflow_pool_tuning() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n generation = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n generation.model_pool = [\"hf:empty\", \"hf:bad-json\", \"hf:ok\"]\n generation.default_model = \"hf:empty\"\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 120,\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"finish_reason\": \"stop\",\n \"visible_chars\": 80,\n },\n },\n {\n \"event_id\": 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"candidate_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"empty_visible_output\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n generation_rec = next(\n item\n for item in summary[\"workflow_recommendations\"]\n if item[\"workflow_id\"] == \"research_generation\"\n )\n by_model = {item[\"model_id\"]: item for item in generation_rec[\"models\"]}\n assert generation_rec[\"required_contracts\"] == [\"candidate_json\"]\n assert generation_rec[\"status\"] == \"needs_attention\"\n assert generation_rec[\"recommended_model_pool\"] == [\"hf:ok\"]\n assert generation_rec[\"recommended_default_model\"] == \"hf:ok\"\n assert by_model[\"hf:empty\"][\"recommendation\"] == \"increase_max_tokens_or_remove\"\n assert by_model[\"hf:bad-json\"][\"recommendation\"] == \"remove_for_contract\"\n assert by_model[\"hf:ok\"][\"recommendation\"] == \"usable\"\n assert \"increase max_tokens\" in by_model[\"hf:empty\"][\"operator_action\"]\n assert \"structurally unreliable\" in by_model[\"hf:bad-json\"][\"operator_action\"]\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "bb0d6f48b7469a13a9623043a77dc58a28d74e53": {"t": 1785769059.9297814, "n": 33932, "h": "fa944cd633b3827f", "c": "from __future__ import annotations\n\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_settings_format_probe_records_schema_success(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"{\\\\\"ok\\\\\":true,'\n b'\\\\\"items\\\\\":[1,2]}\"},\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"prompt_contract\"] == \"strict_json\"\n assert body[\"valid_json\"] is True\n assert body[\"schema_ok\"] is True\n assert body[\"malformed_kind\"] == \"\"\n assert '\"max_tokens\": 128' in str(seen[\"body\"])\n assert \"Return only this compact JSON object\" in str(seen[\"body\"])\n payload = store.events[0][\"payload\"]\n assert payload[\"source\"] == \"format_probe\"\n assert payload[\"prompt_contract\"] == \"strict_json\"\n assert payload[\"valid_json\"] is True\n assert payload[\"schema_ok\"] is True\n assert payload[\"malformed_kind\"] == \"\"\n\n\ndef test_llm_settings_format_probe_records_malformed_output(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"not json\"},'\n b'\"finish_reason\":\"stop\"}]}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"prompt_contract\": \"strict_json\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"valid_json\"] is False\n assert body[\"schema_ok\"] is False\n assert body[\"malformed_kind\"] == \"invalid_json\"\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n assert rows[\"owl/strict-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"owl/strict-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"owl/strict-json\"][\"latest_malformed_kind\"] == \"invalid_json\"\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "6846a31a3084f4cc1c0e6f5336cff635caa36e9b": {"t": 1785769061.3050342, "n": 27726, "h": "fe6811e10036e7cc", "c": "from __future__ import annotations\n\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_settings_model_test_records_visible_output_health(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return (\n b'{\"choices\":[{\"message\":{\"content\":\"\"},\"finish_reason\":\"length\"}],'\n b'\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":12,'\n b'\"completion_tokens_details\":{\"reasoning_tokens\":12}}}'\n )\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(\n {\n \"event_id\": len(self.events) + 1,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n **kwargs,\n }\n )\n return len(self.events)\n\n def event_page(self, **_kwargs: object):\n return self.events, None, False\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"owl/strict-json\",\n provider_id=\"openrouter\",\n label=\"Owl Strict JSON\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n lambda _req, timeout: FakeResponse(),\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"owl/strict-json\",\n \"source\": \"manual\",\n },\n )\n\n assert response.status_code == 200\n body = response.json()\n assert body[\"ok\"] is True\n assert body[\"visible_chars\"] == 0\n assert body[\"finish_reason\"] == \"length\"\n assert body[\"reasoning_tokens\"] == 12\n assert store.events\n payload = store.events[0][\"payload\"]\n assert payload[\"visible_chars\"] == 0\n assert payload[\"finish_reason\"] == \"length\"\n assert payload[\"input_tokens\"] == 5\n assert payload[\"output_tokens\"] == 12\n assert payload[\"reasoning_tokens\"] == 12\n health = client.get(\n \"/control/api/v1/observability/llm-models\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n assert health.status_code == 200\n rows = {row[\"model_id\"]: row for row in health.json()[\"models\"]}\n owl = rows[\"owl/strict-json\"]\n assert owl[\"endpoint_health\"] == \"healthy\"\n assert owl[\"visible_output_health\"] == \"empty\"\n assert owl[\"reasoning_budget_health\"] == \"length_limited\"\n assert health.json()[\"structurally_unhealthy_count\"] == 1\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_llm_model_health_summary_distinguishes_endpoint_and_format_health() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:empty\",\n provider_id=\"synthetic\",\n label=\"Empty Output\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:bad-json\",\n provider_id=\"synthetic\",\n label=\"Bad JSON\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:ok\",\n provider_id=\"synthetic\",\n label=\"OK\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 4,\n \"created_at\": \"2026-06-02T10:02:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:ok\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 65,\n \"checked_at\": \"2026-06-02T10:02:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": True,\n \"schema_ok\": True,\n \"finish_reason\": \"stop\",\n \"visible_chars\": 42,\n },\n },\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-02T10:01:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:bad-json\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 70,\n \"checked_at\": \"2026-06-02T10:01:00Z\",\n \"source\": \"format_probe\",\n \"prompt_contract\": \"strict_json\",\n \"valid_json\": False,\n \"schema_ok\": False,\n \"malformed_kind\": \"invalid_json\",\n \"visible_chars\": 12,\n \"response_preview_redacted\": \"{not json\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-02T10:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:empty\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-02T10:00:00Z\",\n \"finish_reason\": \"length\",\n \"visible_chars\": 0,\n \"reasoning_tokens\": 12,\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"ok\"] is False\n assert summary[\"unhealthy_count\"] == 0\n assert summary[\"structurally_unhealthy_count\"] == 2\n assert rows[\"hf:empty\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:empty\"][\"visible_output_health\"] == \"empty\"\n assert rows[\"hf:empty\"][\"reasoning_budget_health\"] == \"length_limited\"\n assert \"increase output budget\" in rows[\"hf:empty\"][\"operator_action\"]\n assert rows[\"hf:bad-json\"][\"endpoint_health\"] == \"healthy\"\n assert rows[\"hf:bad-json\"][\"format_health\"] == \"degraded\"\n assert rows[\"hf:bad-json\"][\"format_success_rate\"] == 0.0\n assert rows[\"hf:bad-json\"][\"latest_preview\"] == \"{not json\"\n assert rows[\"hf:ok\"][\"format_health\"] == \"healthy\"\n assert rows[\"hf:ok\"][\"workflow_health\"] == \"unmeasured\"\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "b114c0b2d5af52665d73a7510a3e8e9f944808a3": {"t": 1785769062.6403103, "n": 19764, "h": "8793d71bc09a81bb", "c": "from __future__ import annotations\n\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"provider_id\": \"openrouter\",\n \"model_id\": \"moonshotai/kimi-k2.6\",\n \"source\": \"autopilot\",\n },\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert payload[\"source\"] == \"autopilot\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "0b14b261b80f8ebed7a093b5756b534b380b0e95": {"t": 1785769064.0807292, "n": 19544, "h": "f98cea911603b878", "c": "from __future__ import annotations\n\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_untrusted_provider_base_url() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"providers\"][0][\"base_url\"] = \"https://attacker.example/openai/v1\"\n\n with pytest.raises(ValueError, match=\"trusted LLM provider\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_rejects_cross_provider_api_key_env() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"api_key_env\"] = \"SYNTHETIC_API_KEY\"\n\n with pytest.raises(ValueError, match=\"api_key_env for openrouter\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"provider_id\": \"openrouter\", \"model_id\": \"moonshotai/kimi-k2.6\"},\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"provider_id\": \"openrouter\", \"model_id\": \"moonshotai/kimi-k2.6\"},\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n overridden = _resolve_research_cycle_params(\n {\n \"provider_openai_base_url\": \"https://attacker.example/openai/v1\",\n \"provider_base_url\": \"https://attacker.example\",\n }\n )\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert params.provider_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n assert overridden.provider_base_url == \"https://openrouter.ai/api/v1\"\n"}, "596a1d2e3c60e4a6c22a1735881a53d5a0396e43": {"t": 1785769066.472, "n": 18299, "h": "7caa4c468b79bc8f", "c": "from __future__ import annotations\n\nimport tempfile\nimport urllib.error\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(\n config: GateConfig,\n monkeypatch: pytest.MonkeyPatch | None = None,\n store=None,\n) -> TestClient:\n app = FastAPI()\n if monkeypatch is not None and store is not None:\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router._control_plane_store_for_config\",\n lambda _config: store,\n )\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"provider_id\": \"openrouter\", \"model_id\": \"moonshotai/kimi-k2.6\"},\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_llm_settings_model_test_records_scrubbed_health_event(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 429\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"error\":\"rate limited for key or-secret-value\"}'\n\n def close(self) -> None:\n return None\n\n class FakeStore:\n def __init__(self) -> None:\n self.events: list[dict[str, object]] = []\n\n def append_event(self, **kwargs: object) -> int:\n self.events.append(kwargs)\n return len(self.events)\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n store = FakeStore()\n client = _client(config, monkeypatch=monkeypatch, store=store)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n raise urllib.error.HTTPError(\n req.full_url,\n 429,\n \"Too Many Requests\",\n hdrs=None,\n fp=FakeResponse(),\n )\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"provider_id\": \"openrouter\", \"model_id\": \"moonshotai/kimi-k2.6\"},\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is False\n assert \"or-secret-value\" not in response.text\n assert store.events\n event = store.events[0]\n assert event[\"event_type\"] == \"settings.llm.model_test\"\n assert event[\"entity_type\"] == \"llm_model\"\n assert event[\"entity_id\"] == \"openrouter:moonshotai/kimi-k2.6\"\n payload = event[\"payload\"]\n assert payload[\"provider_id\"] == \"openrouter\"\n assert payload[\"model_id\"] == \"moonshotai/kimi-k2.6\"\n assert payload[\"status_code\"] == 429\n assert payload[\"failure_kind\"] == \"rate_limited\"\n assert \"or-secret-value\" not in str(payload)\n\n\ndef test_llm_model_health_summary_marks_recent_failures_and_stale_models() -> None:\n from enoch_control_plane.control_plane.read_models import llm_model_health_summary\n\n settings = default_llm_settings()\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n synthetic.enabled = True\n settings.models = [\n LLMModelSettings(\n model_id=\"hf:healthy\",\n provider_id=\"synthetic\",\n label=\"Healthy\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:gone\",\n provider_id=\"synthetic\",\n label=\"Gone\",\n enabled=True,\n ),\n LLMModelSettings(\n model_id=\"hf:stale\",\n provider_id=\"synthetic\",\n label=\"Stale\",\n enabled=True,\n ),\n ]\n\n class FakeStore:\n def event_page(self, **_kwargs):\n return (\n [\n {\n \"event_id\": 3,\n \"created_at\": \"2026-06-01T19:00:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:gone\",\n \"ok\": False,\n \"status_code\": 404,\n \"failure_kind\": \"model_not_found\",\n \"latency_ms\": 120,\n \"checked_at\": \"2026-06-01T19:00:00Z\",\n },\n },\n {\n \"event_id\": 2,\n \"created_at\": \"2026-06-01T18:59:00Z\",\n \"payload\": {\n \"provider_id\": \"synthetic\",\n \"model_id\": \"hf:healthy\",\n \"ok\": True,\n \"status_code\": 200,\n \"latency_ms\": 80,\n \"checked_at\": \"2026-06-01T18:59:00Z\",\n },\n },\n ],\n None,\n False,\n )\n\n summary = llm_model_health_summary(FakeStore(), settings)\n\n rows = {row[\"model_id\"]: row for row in summary[\"models\"]}\n assert summary[\"unhealthy_count\"] == 2\n assert rows[\"hf:healthy\"][\"status\"] == \"healthy\"\n assert rows[\"hf:healthy\"][\"success_rate\"] == 1.0\n assert rows[\"hf:gone\"][\"status\"] == \"unhealthy\"\n assert rows[\"hf:gone\"][\"latest_failure_kind\"] == \"model_not_found\"\n assert rows[\"hf:stale\"][\"status\"] == \"stale\"\n assert rows[\"hf:stale\"][\"latest\"] is None\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n"}, "164194fde0d521fbe50c35e471176e9077e28139": {"t": 1785769067.6999362, "n": 12383, "h": "d4b3c3a5c602ba7c", "c": "from __future__ import annotations\n\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(config: GateConfig) -> TestClient:\n app = FastAPI()\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_api_recovers_secret_pasted_into_env_field() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config).model_dump(mode=\"json\")\n openrouter = next(\n provider\n for provider in settings[\"providers\"]\n if provider[\"provider_id\"] == \"openrouter\"\n )\n openrouter[\"enabled\"] = True\n openrouter[\"api_key_env\"] = \"sk-or-operator-pasted-key\"\n openrouter[\"api_key_configured\"] = False\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings},\n )\n\n assert response.status_code == 200\n persisted = read_llm_settings(config)\n openrouter_saved = next(\n provider\n for provider in persisted.providers\n if provider.provider_id == \"openrouter\"\n )\n assert openrouter_saved.api_key_env == \"\"\n assert (\n llm_provider_secret_path(config, \"openrouter\").read_text(encoding=\"utf-8\")\n == \"sk-or-operator-pasted-key\\n\"\n )\n assert \"sk-or-operator-pasted-key\" not in response.text\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_llm_settings_model_test_calls_exact_openai_model(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n class FakeResponse:\n status = 200\n\n def __enter__(self):\n return self\n\n def __exit__(self, *_args: object) -> None:\n return None\n\n def read(self, *_args: object) -> bytes:\n return b'{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}'\n\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"moonshotai/kimi-k2.6\",\n provider_id=\"openrouter\",\n label=\"Kimi K2.6\",\n enabled=True,\n weight=90,\n )\n )\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n seen: dict[str, object] = {}\n\n def fake_urlopen(req, timeout: int): # noqa: ANN001 - urllib test double\n seen[\"url\"] = req.full_url\n seen[\"headers\"] = dict(req.header_items())\n seen[\"body\"] = req.data.decode(\"utf-8\")\n seen[\"timeout\"] = timeout\n return FakeResponse()\n\n monkeypatch.setattr(\n \"enoch_control_plane.control_plane.router.urllib.request.urlopen\",\n fake_urlopen,\n )\n\n response = client.post(\n \"/control/api/settings/llm/test\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"provider_id\": \"openrouter\", \"model_id\": \"moonshotai/kimi-k2.6\"},\n )\n\n assert response.status_code == 200\n assert response.json()[\"ok\"] is True\n assert seen[\"url\"] == \"https://openrouter.ai/api/v1/chat/completions\"\n assert seen[\"timeout\"] == 20\n assert \"Bearer or-secret-value\" in str(seen[\"headers\"])\n assert '\"model\": \"moonshotai/kimi-k2.6\"' in str(seen[\"body\"])\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n"}, "1203845cdc3911ab9c8957a484ac57ad5ce877ac": {"t": 1785769069.0572178, "n": 8823, "h": "fc24568be63ea72d", "c": "from __future__ import annotations\n\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _provider_api_key_for_base_url,\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n llm_provider_api_key,\n llm_provider_secret_path,\n read_llm_settings,\n settings_response,\n write_llm_provider_secrets,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(config: GateConfig) -> TestClient:\n app = FastAPI()\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_one_time_provider_secret() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\n \"requested_by\": \"test\",\n \"settings\": settings.model_dump(mode=\"json\"),\n \"provider_secrets\": {\"openrouter\": \"or-secret-value\"},\n },\n )\n\n assert response.status_code == 200\n assert \"or-secret-value\" not in response.text\n providers = response.json()[\"settings\"][\"providers\"]\n openrouter_response = next(\n provider\n for provider in providers\n if provider[\"provider_id\"] == \"openrouter\"\n )\n assert openrouter_response[\"api_key_configured\"] is True\n secret_path = llm_provider_secret_path(config, \"openrouter\")\n assert secret_path.read_text(encoding=\"utf-8\") == \"or-secret-value\\n\"\n assert secret_path.stat().st_mode & 0o777 == 0o600\n assert llm_provider_api_key(config, openrouter) == \"or-secret-value\"\n\n\ndef test_llm_settings_secret_status_uses_secret_file_when_env_missing() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n synthetic = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"synthetic\"\n )\n write_llm_provider_secrets(\n config, {\"synthetic\": \"synthetic-secret\"}, settings=settings\n )\n\n payload = settings_response(settings, config)\n\n provider = next(\n item for item in payload[\"providers\"] if item[\"provider_id\"] == \"synthetic\"\n )\n assert provider[\"api_key_configured\"] is True\n assert \"synthetic-secret\" not in str(payload)\n\n\ndef test_research_provider_api_key_resolves_dashboard_secret_file() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n write_llm_settings(config, settings, updated_by=\"test\")\n write_llm_provider_secrets(\n config, {\"openrouter\": \"or-secret-value\"}, settings=settings\n )\n create_control_plane_router(config, lambda token: None)\n\n assert (\n _provider_api_key_for_base_url(\"https://openrouter.ai/api/v1\")\n == \"or-secret-value\"\n )\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n"}, "37cefc8643baaf70fd196d389dabe0debc3a6842": {"t": 1785769070.4129717, "n": 5691, "h": "833104f1779f95bb", "c": "from __future__ import annotations\n\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n read_llm_settings,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(config: GateConfig) -> TestClient:\n app = FastAPI()\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(\n monkeypatch: pytest.MonkeyPatch,\n) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert (\n response.json()[\"settings\"][\"workflows\"][0][\"default_model\"]\n == \"openrouter/auto\"\n )\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider\n for provider in settings.providers\n if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item\n for item in settings.workflows\n if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n"}, "00c4f1394a7abb31dcf4a2b032c11a947c973e8e": {"t": 1785769071.7635365, "n": 5576, "h": "c4b633fdc932649c", "c": "from __future__ import annotations\n\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom enoch_control_plane.config import GateConfig\nfrom enoch_control_plane.control_plane.router import (\n _resolve_research_cycle_params,\n _resolve_research_provider_model,\n create_control_plane_router,\n)\nfrom enoch_control_plane.llm_settings import (\n LLMModelSettings,\n LLMSettings,\n default_llm_settings,\n read_llm_settings,\n write_llm_settings,\n)\n\n\nTOKEN = \"test-token\"\n\n\ndef _config(tmp: str) -> GateConfig:\n root = Path(tmp) / \"projects\"\n root.mkdir(parents=True, exist_ok=True)\n return GateConfig(\n state_dir=str(Path(tmp) / \"state\"),\n project_root=str(root),\n dispatch_script_path=str(Path(tmp) / \"dispatch.sh\"),\n control_api_bearer_token=TOKEN,\n completion_callback_url=\"http://example.invalid/callback\",\n completion_callback_token=\"unused\",\n )\n\n\ndef _client(config: GateConfig) -> TestClient:\n app = FastAPI()\n app.include_router(create_control_plane_router(config, lambda token: None))\n return TestClient(app)\n\n\ndef test_default_llm_settings_include_provider_and_workflow_pools() -> None:\n settings = default_llm_settings()\n\n assert {provider.provider_id for provider in settings.providers} >= {\n \"synthetic\",\n \"openrouter\",\n \"openai\",\n \"anthropic\",\n }\n workflows = {workflow.workflow_id: workflow for workflow in settings.workflows}\n assert workflows[\"research_generation\"].model_pool\n assert workflows[\"paper_writing\"].model_pool\n assert workflows[\"research_review\"].model_pool\n\n\ndef test_llm_settings_reject_unknown_workflow_model() -> None:\n settings = default_llm_settings().model_dump(mode=\"json\")\n settings[\"workflows\"][0][\"model_pool\"].append(\"missing-model\")\n\n with pytest.raises(ValueError, match=\"unknown models\"):\n LLMSettings.model_validate(settings)\n\n\ndef test_llm_settings_api_does_not_expose_secret_values(monkeypatch: pytest.MonkeyPatch) -> None:\n with tempfile.TemporaryDirectory() as tmp:\n monkeypatch.setenv(\"SYNTHETIC_API_KEY\", \"secret-value-that-must-not-return\")\n client = _client(_config(tmp))\n\n response = client.get(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n )\n\n assert response.status_code == 200\n body = response.json()\n providers = body[\"settings\"][\"providers\"]\n synthetic = next(\n provider for provider in providers if provider[\"provider_id\"] == \"synthetic\"\n )\n assert synthetic[\"api_key_configured\"] is True\n assert \"secret-value-that-must-not-return\" not in response.text\n\n\ndef test_llm_settings_api_persists_valid_updates() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n client = _client(config)\n settings = default_llm_settings(config)\n openrouter = next(\n provider for provider in settings.providers if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n research = next(\n workflow\n for workflow in settings.workflows\n if workflow.workflow_id == \"research_generation\"\n )\n research.provider_ids = [\"openrouter\"]\n research.model_pool = [\"openrouter/auto\"]\n research.default_model = \"openrouter/auto\"\n\n response = client.post(\n \"/control/api/settings/llm\",\n headers={\"Authorization\": f\"Bearer {TOKEN}\"},\n json={\"requested_by\": \"test\", \"settings\": settings.model_dump(mode=\"json\")},\n )\n\n assert response.status_code == 200\n assert response.json()[\"settings\"][\"workflows\"][0][\"default_model\"] == \"openrouter/auto\"\n persisted = read_llm_settings(config)\n assert persisted.workflows[0].default_model == \"openrouter/auto\"\n\n\ndef test_research_provider_selection_uses_persisted_settings() -> None:\n with tempfile.TemporaryDirectory() as tmp:\n config = _config(tmp)\n settings = default_llm_settings(config)\n openrouter = next(\n provider for provider in settings.providers if provider.provider_id == \"openrouter\"\n )\n openrouter.enabled = True\n settings.models.append(\n LLMModelSettings(\n model_id=\"openrouter/auto\",\n provider_id=\"openrouter\",\n label=\"OpenRouter Auto\",\n enabled=True,\n weight=1,\n )\n )\n workflow = next(\n item for item in settings.workflows if item.workflow_id == \"research_generation\"\n )\n workflow.provider_ids = [\"openrouter\"]\n workflow.model_pool = [\"openrouter/auto\"]\n workflow.default_model = \"openrouter/auto\"\n write_llm_settings(config, settings, updated_by=\"test\")\n create_control_plane_router(config, lambda token: None)\n\n model, allowed_models = _resolve_research_provider_model({})\n params = _resolve_research_cycle_params({})\n\n assert model == \"openrouter/auto\"\n assert allowed_models == [\"openrouter/auto\"]\n assert params.provider_openai_base_url == \"https://openrouter.ai/api/v1\"\n"}}}} |