OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
39 KiB
JSON
1 line
39 KiB
JSON
{"repo": "brosax/webui", "files": {"tests/test_model_resolver.py": {"e1a27ebf8a13d6b8fa3b8edaa87f00c553e6c262": {"t": 1785846972.918118, "n": 24429, "h": "16b27a1142db3d90", "c": "\"\"\"\nTests for resolve_model_provider() model routing logic.\nVerifies that model IDs are correctly resolved to (model, provider, base_url)\ntuples for different provider configurations.\n\"\"\"\nimport pytest\nimport api.config as config\n\n\ndef _resolve_with_config(model_id, provider=None, base_url=None, default=None, custom_providers=None):\n \"\"\"Helper: temporarily set config.cfg model/custom provider sections, call resolve, restore.\"\"\"\n old_cfg = dict(config.cfg)\n model_cfg = {}\n if provider:\n model_cfg['provider'] = provider\n if base_url:\n model_cfg['base_url'] = base_url\n if default:\n model_cfg['default'] = default\n config.cfg['model'] = model_cfg if model_cfg else {}\n if custom_providers is not None:\n config.cfg['custom_providers'] = custom_providers\n try:\n return config.resolve_model_provider(model_id)\n finally:\n config.cfg.clear()\n config.cfg.update(old_cfg)\n\n\n# ── OpenRouter prefix handling ────────────────────────────────────────────\n\ndef test_openrouter_free_keeps_full_path():\n \"\"\"openrouter/free must NOT be stripped to 'free' when provider is openrouter.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'openrouter/free', provider='openrouter',\n base_url='https://openrouter.ai/api/v1',\n )\n assert model == 'openrouter/free', f\"Expected 'openrouter/free', got '{model}'\"\n assert provider == 'openrouter'\n\n\ndef test_openrouter_model_with_provider_prefix():\n \"\"\"anthropic/claude-sonnet-4.6 via openrouter keeps full path.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'anthropic/claude-sonnet-4.6', provider='openrouter',\n base_url='https://openrouter.ai/api/v1',\n )\n assert model == 'anthropic/claude-sonnet-4.6'\n assert provider == 'openrouter'\n\n\n# ── Direct provider prefix stripping ─────────────────────────────────────\n\ndef test_anthropic_prefix_stripped_for_direct_api():\n \"\"\"anthropic/claude-sonnet-4.6 strips prefix when provider is anthropic.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'anthropic/claude-sonnet-4.6', provider='anthropic',\n )\n assert model == 'claude-sonnet-4.6'\n assert provider == 'anthropic'\n\n\ndef test_openai_prefix_stripped_for_direct_api():\n \"\"\"openai/gpt-5.4-mini strips prefix when provider is openai.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'openai/gpt-5.4-mini', provider='openai',\n )\n assert model == 'gpt-5.4-mini'\n assert provider == 'openai'\n\n\n# ── Cross-provider routing ───────────────────────────────────────────────\n\ndef test_cross_provider_routes_through_openrouter():\n \"\"\"Picking openai model when config is anthropic routes via openrouter.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'openai/gpt-5.4-mini', provider='anthropic',\n )\n assert model == 'openai/gpt-5.4-mini'\n assert provider == 'openrouter'\n assert base_url is None # openrouter uses its own endpoint\n\n\n# ── Bare model names ─────────────────────────────────────────────────────\n\ndef test_bare_model_uses_config_provider():\n \"\"\"A model name without / uses the config provider and base_url.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'gemma-4-26B', provider='custom',\n base_url='http://192.168.1.160:4000',\n )\n assert model == 'gemma-4-26B'\n assert provider == 'custom'\n assert base_url == 'http://192.168.1.160:4000'\n\n\ndef test_empty_model_returns_config_defaults():\n \"\"\"Empty model string returns config provider and base_url.\"\"\"\n model, provider, base_url = _resolve_with_config(\n '', provider='anthropic',\n )\n assert model == ''\n assert provider == 'anthropic'\n\n\n# ── @provider:model hint routing (Issue #138 v2) ────────────────────────\n\ndef test_provider_hint_routes_to_specific_provider():\n \"\"\"@minimax:MiniMax-M2.7 routes to minimax provider directly.\"\"\"\n model, provider, base_url = _resolve_with_config(\n '@minimax:MiniMax-M2.7', provider='anthropic',\n )\n assert model == 'MiniMax-M2.7'\n assert provider == 'minimax'\n assert base_url is None # resolve_runtime_provider will fill this\n\n\ndef test_provider_hint_zai():\n \"\"\"@zai:GLM-5 routes to zai provider directly.\"\"\"\n model, provider, base_url = _resolve_with_config(\n '@zai:GLM-5', provider='openai',\n )\n assert model == 'GLM-5'\n assert provider == 'zai'\n\n\ndef test_provider_hint_deepseek():\n \"\"\"@deepseek:deepseek-chat routes to deepseek provider.\"\"\"\n model, provider, base_url = _resolve_with_config(\n '@deepseek:deepseek-chat', provider='anthropic',\n )\n assert model == 'deepseek-chat'\n assert provider == 'deepseek'\n\n\ndef test_slash_prefix_non_default_still_routes_openrouter():\n \"\"\"minimax/MiniMax-M2.7 (old format) still routes through openrouter.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'minimax/MiniMax-M2.7', provider='anthropic',\n )\n assert model == 'minimax/MiniMax-M2.7'\n assert provider == 'openrouter'\n\n\ndef test_custom_provider_model_with_slash_routes_to_named_custom_provider():\n \"\"\"Slash-containing custom endpoint model IDs must not be mistaken for OpenRouter models.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'google/gemma-4-26b-a4b',\n provider='openrouter',\n base_url='https://openrouter.ai/api/v1',\n custom_providers=[{\n 'name': 'Local LM Studio',\n 'base_url': 'http://lmstudio.local:1234/v1',\n 'model': 'google/gemma-4-26b-a4b',\n }],\n )\n assert model == 'google/gemma-4-26b-a4b'\n assert provider == 'custom:local-lm-studio'\n assert base_url == 'http://lmstudio.local:1234/v1'\n\n\ndef test_custom_provider_models_dict_routes_to_named_custom_provider():\n \"\"\"Models listed only under custom_providers[].models still route to that endpoint.\"\"\"\n model, provider, base_url = _resolve_with_config(\n 'sensenova-6.7-flash-lite',\n provider='xiaomi',\n custom_providers=[{\n 'name': 'LiteLLM Proxy',\n 'base_url': 'http://127.0.0.1:8080/v1',\n 'model': 'deepseek-v4-flash',\n 'models': {\n 'deepseek-v4-flash': {},\n 'sensenova-6.7-flash-lite': {},\n },\n }],\n )\n assert model == 'sensenova-6.7-flash-lite'\n assert provider == 'custom:litellm-proxy'\n assert base_url == 'http://127.0.0.1:8080/v1'\n\n\n# ── Issue #1922: default model shadowed by overlapping custom_providers[] ──\n\ndef test_default_model_not_shadowed_by_overlapping_custom_provider():\n r'''Regression test for #1922.\n\n When the active provider is an explicit non-custom provider (e.g. ai-gateway,\n openrouter, xiaomi) AND the requested model_id matches the configured default\n model, the active provider's base_url must take precedence over an overlapping\n custom_providers[] entry. Otherwise the WebUI routes to 'custom:<name>' with\n the wrong endpoint, causing 401 errors.\n\n This test mirrors the reported scenario:\n - provider: ai-gateway\n - base_url: https://api.ai-gateway.example/v1\n - default: gpt-5.4\n - An overlapping custom_providers[] entry with the same default model\n\n Expected: active provider (ai-gateway) wins over custom provider.\n '''\n model, provider, base_url = _resolve_with_config(\n 'gpt-5.4',\n provider='ai-gateway',\n base_url='https://api.ai-gateway.example/v1',\n default='gpt-5.4',\n custom_providers=[{\n 'name': 'My Custom Endpoint',\n 'base_url': 'http://localhost:8080/v1',\n 'model': 'gpt-5.4',\n }],\n )\n assert model == 'gpt-5.4', f'Expected model=gpt-5.4, got {model!r}'\n assert provider == 'ai-gateway', f'Expected provider=ai-gateway, got {provider!r}'\n assert base_url == 'https://api.ai-gateway.example/v1', f'Expected base_url from active provider, got {base_url!r}'\n\n\ndef test_default_model_shadowed_with_xiaomi_provider():\n r'''Same regression test with provider=xiaomi instead of ai-gateway.'''\n model, provider, base_url = _resolve_with_config(\n 'deepseek-v4-flash',\n provider='xiaomi',\n default='deepseek-v4-flash',\n custom_providers=[{\n 'name': 'LiteLLM Proxy',\n 'base_url': 'http://127.0.0.1:8080/v1',\n 'model': 'deepseek-v4-flash',\n }],\n )\n assert model == 'deepseek-v4-flash'\n assert provider == 'xiaomi'\n assert base_url is None # xiaomi has no config base_url in this test\n\n\n# ── get_available_models() @provider: hint behaviour ──────────────────────\n\n\n@pytest.fixture(autouse=True)\ndef _isolate_models_cache():\n \"\"\"Invalidate the models TTL cache before and after every test in this file.\n\n Several helpers here mutate ``config.cfg`` in-memory and call\n ``get_available_models()``. Without this guard, a prior test that called\n ``get_available_models()`` leaves a 60-second TTL cache entry; the next\n test that mutates cfg and calls the function gets a cache hit instead of\n running the function body, causing silently wrong results (e.g. the\n ``test_custom_endpoint_uses_model_config_api_key_for_model_discovery``\n ``KeyError: 'auth'`` on CI where ``urlopen`` is never reached).\n \"\"\"\n try:\n config.invalidate_models_cache()\n except Exception:\n pass\n yield\n try:\n config.invalidate_models_cache()\n except Exception:\n pass\n\n\ndef _available_models_with_provider(provider):\n \"\"\"Helper: temporarily set active_provider in config.\"\"\"\n old_cfg = dict(config.cfg)\n config.cfg['model'] = {'provider': provider}\n try:\n return config.get_available_models()\n finally:\n config.cfg.clear()\n config.cfg.update(old_cfg)\n\n\ndef test_non_default_provider_models_use_hint_prefix():\n \"\"\"With anthropic as default, minimax model IDs should use @minimax: prefix.\"\"\"\n result = _available_models_with_provider('anthropic')\n groups = {g['provider']: g['models'] for g in result['groups']}\n if 'MiniMax' in groups:\n for m in groups['MiniMax']:\n assert m['id'].startswith('@minimax:'), (\n f\"Expected @minimax: prefix, got: {m['id']!r}\"\n )\n\n\ndef test_no_duplicate_when_default_model_is_prefixed():\n \"\"\"Issue #147 Bug 2: 'anthropic/claude-opus-4.6' as default_model must not\n inject a duplicate alongside the existing bare 'claude-opus-4.6' entry in\n the same provider group.\"\"\"\n import api.config as _cfg\n old_cfg = dict(_cfg.cfg)\n _cfg.cfg['model'] = {\n 'provider': 'anthropic',\n 'default': 'anthropic/claude-opus-4.6',\n }\n try:\n result = _cfg.get_available_models()\n norm = lambda mid: mid.split('/', 1)[-1] if '/' in mid else mid\n # Check each group individually: no group should have two entries that\n # normalize to the same bare model name\n for g in result['groups']:\n bare_ids = [norm(m['id']) for m in g['models']]\n duplicates = [mid for mid in set(bare_ids) if bare_ids.count(mid) > 1]\n assert not duplicates, (\n f\"Provider group '{g['provider']}' has duplicate models after normalization: \"\n f\"{duplicates}\\nFull group: {[m['id'] for m in g['models']]}\"\n )\n finally:\n _cfg.cfg.clear()\n _cfg.cfg.update(old_cfg)\n\n\ndef test_default_provider_models_not_prefixed(monkeypatch):\n \"\"\"The active provider's models remain bare (no @prefix added).\"\"\"\n import api.config as _cfg\n monkeypatch.setattr(_cfg, \"_read_live_provider_model_ids\", lambda pid: [\"claude-sonnet-5.0\"] if pid == \"anthropic\" else [])\n result = _available_models_with_provider('anthropic')\n groups = {g['provider']: g['models'] for g in result['groups']}\n if 'Anthropic' in groups:\n returned_ids = {m['id'] for m in groups['Anthropic']}\n assert \"claude-sonnet-5.0\" in returned_ids\n assert not any(mid.startswith('@anthropic:') for mid in returned_ids), returned_ids\n\n\n# ── get_available_models(): phantom \"Custom\" group regression ─────────────\n#\n# When the user has model.provider set to a real provider (e.g. openai-codex)\n# AND a model.base_url set, hermes_cli reports the 'custom' pseudo-provider as\n# authenticated. The WebUI picker must NOT build a separate \"Custom\" group in\n# that case — the base_url belongs to the active provider.\n\ndef _available_models_with_full_cfg(provider, default, base_url):\n \"\"\"Helper: set model.provider, model.default, model.base_url at once.\n\n Clears model-override env vars (HERMES_MODEL, OPENAI_MODEL, LLM_MODEL)\n during the call so the real hermes profile environment doesn't leak into\n the test and override the fixture's default model.\n \"\"\"\n import os\n import api.config as _cfg\n old_cfg = dict(_cfg.cfg)\n _cfg.cfg['model'] = {\n 'provider': provider,\n 'default': default,\n 'base_url': base_url,\n }\n try:\n _cfg._cfg_mtime = _cfg.Path(_cfg._get_config_path()).stat().st_mtime\n except Exception:\n # No config.yaml on this machine (e.g. CI); pin to 0.0 so the mtime check\n # inside get_available_models() sees 0.0 == 0.0 and doesn't call reload_config(),\n # which would overwrite the in-memory cfg we just set up.\n _cfg._cfg_mtime = 0.0\n # Clear model-override env vars to prevent the real profile from leaking in\n _model_env_keys = ('HERMES_MODEL', 'OPENAI_MODEL', 'LLM_MODEL')\n _saved_env = {k: os.environ.pop(k, None) for k in _model_env_keys}\n try:\n return _cfg.get_available_models()\n finally:\n _cfg.cfg.clear()\n _cfg.cfg.update(old_cfg)\n for k, v in _saved_env.items():\n if v is not None:\n os.environ[k] = v\n\n\ndef test_no_phantom_custom_group_when_active_provider_is_set(monkeypatch):\n \"\"\"Issue: with provider=openai-codex + base_url set, gpt-5.4 was landing\n under a phantom \"Custom\" group instead of the \"OpenAI Codex\" group.\"\"\"\n import sys, types\n\n # Force hermes_cli to report both the real provider and the phantom\n # 'custom' as authenticated, simulating what list_available_providers()\n # returns when base_url is configured.\n fake_mod = types.ModuleType('hermes_cli.models')\n fake_mod.list_available_providers = lambda: [\n {'id': 'openai-codex', 'authenticated': True},\n {'id': 'custom', 'authenticated': True},\n ]\n fake_auth = types.ModuleType('hermes_cli.auth')\n fake_auth.get_auth_status = lambda pid: {'key_source': 'env'}\n monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod)\n monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth)\n\n result = _available_models_with_full_cfg(\n provider='openai-codex',\n default='gpt-5.4',\n base_url='https://chatgpt.com/backend-api/codex',\n )\n group_names = [g['provider'] for g in result['groups']]\n assert 'Custom' not in group_names, (\n f\"Phantom 'Custom' group present; full groups: {group_names}\"\n )\n\n\ndef test_default_model_lands_under_active_provider_group(monkeypatch):\n \"\"\"The configured default_model must appear under the active provider's\n display group, even when the model isn't in _PROVIDER_MODELS[provider]\n AND the active provider isn't the alphabetical first detected provider.\n\n Regression guard for a hyphen-vs-space bug in the \"ensure default_model\n appears\" post-pass: the substring check `active_provider.lower() in\n g.get('provider', '').lower()` was failing for 'openai-codex' vs\n display name 'OpenAI Codex' (hyphen vs. space), silently falling back\n to groups[0] — which, when another provider sorted earlier\n alphabetically (e.g. 'anthropic'), placed gpt-5.4 in the WRONG group.\n \"\"\"\n import sys, types\n fake_mod = types.ModuleType('hermes_cli.models')\n fake_mod.list_available_providers = lambda: [\n {'id': 'anthropic', 'authenticated': True}, # sorts before openai-codex\n {'id': 'openai-codex', 'authenticated': True},\n {'id': 'custom', 'authenticated': True},\n ]\n fake_auth = types.ModuleType('hermes_cli.auth')\n fake_auth.get_auth_status = lambda pid: {'key_source': 'env'}\n monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod)\n monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth)\n\n result = _available_models_with_full_cfg(\n provider='openai-codex',\n default='gpt-5.4',\n base_url='https://chatgpt.com/backend-api/codex',\n )\n groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']}\n assert 'OpenAI Codex' in groups, f\"OpenAI Codex group missing: {list(groups)}\"\n norm = lambda mid: mid.split('/', 1)[-1].split(':', 1)[-1]\n assert 'gpt-5.4' in {norm(mid) for mid in groups['OpenAI Codex']}, (\n f\"gpt-5.4 not in OpenAI Codex group; contents: {groups['OpenAI Codex']}\"\n )\n # And crucially, it must NOT have landed in the alphabetically-first\n # group (Anthropic) via the fallback path.\n assert 'gpt-5.4' not in {norm(mid) for mid in groups.get('Anthropic', [])}, (\n f\"gpt-5.4 leaked into Anthropic group via fallback: {groups.get('Anthropic')}\"\n )\n\n\ndef test_unknown_providers_do_not_inherit_default_model(monkeypatch):\n \"\"\"Detected providers without their own model catalog must not be filled\n with the global default_model placeholder.\n\n Regression guard for the bug where unknown providers ended up showing\n gpt-5.4-mini even though those providers do not serve it. Minimax-Cn is\n now known and should show its own catalog instead.\n \"\"\"\n import sys, types\n\n fake_mod = types.ModuleType('hermes_cli.models')\n fake_mod.list_available_providers = lambda: [\n {'id': 'openai-codex', 'authenticated': True},\n {'id': 'alibaba', 'authenticated': True},\n {'id': 'minimax-cn', 'authenticated': True},\n ]\n fake_auth = types.ModuleType('hermes_cli.auth')\n fake_auth.get_auth_status = lambda pid: {'key_source': 'env'}\n monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod)\n monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth)\n\n result = _available_models_with_full_cfg(\n provider='openai-codex',\n default='gpt-5.4-mini',\n base_url='',\n )\n groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']}\n norm = lambda mid: mid.split('/', 1)[-1].split(':', 1)[-1]\n\n assert 'Alibaba' not in groups, (\n f\"Alibaba should not inherit the default model placeholder: {groups}\"\n )\n assert 'MiniMax (China)' in groups, (\n f\"Minimax-Cn should render its own static catalog: {groups}\"\n )\n assert not any(\n norm(mid) == 'gpt-5.4-mini'\n for mid in groups.get('Alibaba', []) + groups.get('MiniMax (China)', [])\n ), (\n f\"Unknown provider groups still inherited the default model: {groups}\"\n )\n\n\ndef test_custom_endpoint_uses_model_config_api_key_for_model_discovery(monkeypatch):\n \"\"\"Custom endpoint model discovery must use model.api_key from config.yaml,\n not only environment variables, otherwise the dropdown collapses to the\n default model when /v1/models requires auth.\"\"\"\n import json as _json\n import api.config as _cfg\n\n old_cfg = dict(_cfg.cfg)\n _cfg.cfg['model'] = {\n 'provider': 'custom',\n 'default': 'gpt-5.4',\n 'base_url': 'https://example.test/v1',\n 'api_key': 'sk-test-model-key',\n }\n try:\n _cfg._cfg_mtime = _cfg.Path(_cfg._get_config_path()).stat().st_mtime\n except Exception:\n # No config.yaml on this machine (e.g. CI); pin to 0.0 so the mtime check\n # inside get_available_models() sees 0.0 == 0.0 and skips reload_config().\n _cfg._cfg_mtime = 0.0\n _cfg.cfg.pop('providers', None)\n\n captured = {}\n\n class _Resp:\n def read(self):\n return _json.dumps({'data': [{'id': 'gpt-5.2', 'name': 'GPT-5.2'}]}).encode('utf-8')\n def __enter__(self):\n return self\n def __exit__(self, exc_type, exc, tb):\n return False\n\n def _fake_urlopen(req, timeout=10):\n url = getattr(req, 'full_url', '')\n if 'example.test' in url:\n captured['auth'] = req.get_header('Authorization')\n captured['ua'] = req.get_header('User-agent')\n return _Resp()\n\n monkeypatch.setattr('urllib.request.urlopen', _fake_urlopen)\n monkeypatch.setattr('socket.getaddrinfo', lambda *a, **k: [])\n monkeypatch.delenv('OPENAI_API_KEY', raising=False)\n monkeypatch.delenv('HERMES_API_KEY', raising=False)\n monkeypatch.delenv('HERMES_OPENAI_API_KEY', raising=False)\n monkeypatch.delenv('LOCAL_API_KEY', raising=False)\n monkeypatch.delenv('OPENROUTER_API_KEY', raising=False)\n monkeypatch.delenv('API_KEY', raising=False)\n try:\n result = _cfg.get_available_models()\n finally:\n _cfg.cfg.clear()\n _cfg.cfg.update(old_cfg)\n\n assert captured['auth'] == 'Bearer sk-test-model-key'\n assert captured['ua'] == 'OpenAI/Python 1.0'\n groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']}\n assert 'Custom' in groups\n # Model ID may be prefixed with @provider: due to cross-provider dedup (#1228)\n assert any('gpt-5.2' in m for m in groups['Custom']), f'gpt-5.2 not found in Custom: {groups}'\n\n\n# -- Issue #230: custom provider with slash model name -----------------------\n\ndef test_custom_endpoint_slash_model_routes_to_custom_not_openrouter():\n \"\"\"Regression test for #230, updated for #1625.\n\n When provider=custom (or any non-openrouter provider) and base_url is set,\n a model name containing a slash (e.g. google/gemma-4-26b-a4b) must NOT be\n rerouted to OpenRouter -- it should stay on the configured custom endpoint.\n\n #1625 layered an additional rule on top: a base_url pointing at a loopback\n or private-IP host is treated as a local model server (LM Studio, Ollama,\n llama.cpp, vLLM, TabbyAPI), which register models under their full\n HuggingFace path. On such hosts the prefix is now PRESERVED. The original\n #433 strip behaviour still applies on public hosts (real OpenAI-compatible\n proxies like LiteLLM at https://litellm.example.com/v1).\n \"\"\"\n # --- custom provider with slash model name should NOT go to openrouter ---\n model, provider, base_url = _resolve_with_config(\n 'google/gemma-4-26b-a4b',\n provider='custom',\n base_url='http://127.0.0.1:1234/v1',\n default='google/gemma-4-26b-a4b',\n )\n assert provider.startswith('custom'), (\n \"Expected provider starting with 'custom', got '{}'. \"\n \"Slash in model name should NOT trigger OpenRouter rerouting when base_url is set.\".format(provider)\n )\n assert base_url == 'http://127.0.0.1:1234/v1', (\n \"Expected base_url 'http://127.0.0.1:1234/v1', got '{}'.\".format(base_url)\n )\n # #1625 (supersedes the v0.50 #433 strip-on-custom rule for loopback hosts):\n # 127.0.0.1 base_url is almost certainly a local LM Studio / Ollama / etc.,\n # which keys models on the full HuggingFace path. Preserve the prefix.\n assert model == 'google/gemma-4-26b-a4b', (\n \"Model name prefix must be PRESERVED on loopback base_url (#1625), got '{}'.\".format(model)\n )\n\n # --- public-host openai-compatible proxy STILL strips per #433 ----------\n model2, provider2, base_url2 = _resolve_with_config(\n 'google/gemma-4-26b-a4b',\n provider='openai',\n base_url='https://litellm.example.com/v1',\n default='google/gemma-4-26b-a4b',\n )\n assert model2 == 'gemma-4-26b-a4b', (\n \"Public-host OpenAI-compat proxy must still strip prefix per #433, got '{}'.\".format(model2)\n )\n\n # --- openrouter with slash model name MUST still route to openrouter -----\n model_or, provider_or, _ = _resolve_with_config(\n 'google/gemma-4-26b-a4b',\n provider='openrouter',\n base_url='https://openrouter.ai/api/v1',\n default='google/gemma-4-26b-a4b',\n )\n assert provider_or == 'openrouter', (\n \"Expected provider 'openrouter', got '{}'. \"\n \"Slash model via openrouter provider must still resolve to openrouter.\".format(provider_or)\n )\n assert model_or == 'google/gemma-4-26b-a4b', (\n \"Model name should be preserved for openrouter, got '{}'.\".format(model_or)\n )\n"}}, "tests/test_issue1538_nous_live_catalog.py": {"e1a27ebf8a13d6b8fa3b8edaa87f00c553e6c262": {"t": 1786068241.8234766, "n": 13352, "h": "da36b4b3e3858c5f", "c": "\"\"\"Regression tests for #1538 — Nous Portal model picker should live-fetch\nthe full catalog (~30 models) instead of returning the four-entry static list.\n\nBackground\n----------\nSettings → Default Model showed only four Nous models (Claude Opus 4.6, Claude\nSonnet 4.6, GPT-5.4 Mini, Gemini 3.1 Pro Preview) because\n``_build_available_models_uncached()`` fell through to the generic\n``pid in _PROVIDER_MODELS`` branch and returned ``copy.deepcopy(_PROVIDER_MODELS[\"nous\"])``.\nThe actual Nous Portal catalog has ~30 models live — including the latest\nAnthropic 4.7 family, GPT-5.5, Gemini 3.1 Pro/Flash, Kimi K2.6, MiniMax M2.7,\nseveral Xiaomi/Tencent/StepFun entries.\n\nFix\n---\nA dedicated ``elif pid == \"nous\":`` branch in ``_build_available_models_uncached()``\nmirroring the Ollama Cloud pattern: live-fetch via\n``hermes_cli.models.provider_model_ids(\"nous\")``, prefix every id with ``@nous:``\nto match the existing routing convention, fall back to the curated static\nlist when ``hermes_cli`` is unavailable.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport sys\nimport types\n\nimport api.config as config\nimport api.profiles as profiles\n\n\n# Sample Nous catalog used in the live-fetch test. Mirrors the shape returned\n# by hermes_cli.models.provider_model_ids(\"nous\") (see #1538 issue body).\nSAMPLE_NOUS_LIVE_IDS = [\n \"moonshotai/kimi-k2.6\",\n \"xiaomi/mimo-v2.5-pro\",\n \"anthropic/claude-opus-4.7\",\n \"anthropic/claude-opus-4.6\",\n \"anthropic/claude-sonnet-4.6\",\n \"anthropic/claude-haiku-4.5\",\n \"openai/gpt-5.5\",\n \"openai/gpt-5.4-mini\",\n \"openai/gpt-5.3-codex\",\n \"google/gemini-3-pro-preview\",\n \"google/gemini-3.1-pro-preview\",\n \"google/gemini-3.1-flash-lite-preview\",\n \"qwen/qwen3.5-plus-02-15\",\n \"minimax/minimax-m2.7\",\n \"z-ai/glm-5.1\",\n \"x-ai/grok-4.20-beta\",\n \"tencent/hy3-preview\",\n \"stepfun/step-3.5-flash\",\n \"nvidia/nemotron-3-super-120b-a12b\",\n \"arcee-ai/trinity-large-thinking\",\n]\n\n\ndef _install_fake_hermes_cli(monkeypatch, *, nous_ids=None, raise_on_lookup=False):\n \"\"\"Install fake ``hermes_cli`` modules so detection sees Nous as authenticated\n and ``provider_model_ids(\"nous\")`` returns the desired catalog.\n\n Mirrors :func:`tests.test_issue1420_lmstudio_provider_env_var._install_fake_hermes_cli`\n but specialised for Nous detection (Nous is OAuth so the env-var path\n is not used — we drive detection via ``hermes_cli.auth.list_auth_providers``).\n \"\"\"\n fake_pkg = types.ModuleType(\"hermes_cli\")\n fake_pkg.__path__ = []\n\n fake_models = types.ModuleType(\"hermes_cli.models\")\n fake_models.list_available_providers = lambda: []\n if raise_on_lookup:\n def _raise(_pid):\n raise RuntimeError(\"simulated hermes_cli failure\")\n fake_models.provider_model_ids = _raise\n else:\n ids = list(nous_ids) if nous_ids is not None else []\n fake_models.provider_model_ids = lambda pid: ids if pid == \"nous\" else []\n\n fake_auth = types.ModuleType(\"hermes_cli.auth\")\n\n def _list_auth_providers():\n return [{\"id\": \"nous\", \"authenticated\": True}]\n\n def _get_auth_status(pid):\n return {\"logged_in\": True, \"key_source\": \"\"} if pid == \"nous\" else {}\n\n fake_auth.list_auth_providers = _list_auth_providers\n fake_auth.get_auth_status = _get_auth_status\n\n monkeypatch.setitem(sys.modules, \"hermes_cli\", fake_pkg)\n monkeypatch.setitem(sys.modules, \"hermes_cli.models\", fake_models)\n monkeypatch.setitem(sys.modules, \"hermes_cli.auth\", fake_auth)\n monkeypatch.delitem(sys.modules, \"agent.credential_pool\", raising=False)\n monkeypatch.delitem(sys.modules, \"agent\", raising=False)\n\n config.invalidate_models_cache()\n\n\ndef _swap_in_test_config(extra_cfg):\n \"\"\"Snapshot config.cfg, replace with a minimal test config; return restore-fn.\"\"\"\n old_cfg = dict(config.cfg)\n old_mtime = config._cfg_mtime\n config.cfg.clear()\n config.cfg[\"model\"] = {}\n config.cfg.update(extra_cfg)\n try:\n config._cfg_mtime = config.Path(config._get_config_path()).stat().st_mtime\n except Exception:\n config._cfg_mtime = 0.0\n\n def _restore():\n config.cfg.clear()\n config.cfg.update(old_cfg)\n config._cfg_mtime = old_mtime\n\n return _restore\n\n\ndef _scrub_provider_env(monkeypatch):\n \"\"\"Drop every provider env var so detection only sees what we install\n via the fake hermes_cli stubs (not unrelated keys leaked from the runner).\"\"\"\n for var in (\n \"ANTHROPIC_API_KEY\", \"OPENAI_API_KEY\", \"GOOGLE_API_KEY\", \"GEMINI_API_KEY\",\n \"DEEPSEEK_API_KEY\", \"XAI_API_KEY\", \"GROQ_API_KEY\",\n \"MISTRAL_API_KEY\", \"OPENROUTER_API_KEY\",\n \"OLLAMA_CLOUD_API_KEY\", \"OLLAMA_API_KEY\",\n \"GLM_API_KEY\", \"KIMI_API_KEY\", \"MOONSHOT_API_KEY\",\n \"MINIMAX_API_KEY\", \"MINIMAX_CN_API_KEY\",\n \"XIAOMI_API_KEY\",\n \"OPENCODE_ZEN_API_KEY\", \"OPENCODE_GO_API_KEY\",\n \"NOUS_API_KEY\", \"NVIDIA_API_KEY\", \"LM_API_KEY\", \"LMSTUDIO_API_KEY\",\n ):\n monkeypatch.delenv(var, raising=False)\n\n\nclass TestNousLiveCatalog:\n \"\"\"When the Nous live catalog is available, the dropdown must surface it\n in full (>=20 entries) — not the four-entry static fallback (#1538).\"\"\"\n\n def test_nous_models_live_fetch_when_hermes_cli_available(self, monkeypatch, tmp_path):\n _scrub_provider_env(monkeypatch)\n _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS)\n monkeypatch.setattr(profiles, \"get_active_hermes_home\", lambda: tmp_path)\n\n restore = _swap_in_test_config({\"model\": {\"provider\": \"nous\"}})\n try:\n data = config.get_available_models()\n nous_groups = [g for g in data.get(\"groups\", []) if g.get(\"provider_id\") == \"nous\"]\n assert len(nous_groups) == 1, (\n f\"Expected exactly one Nous group, got {len(nous_groups)}: \"\n f\"{[g.get('provider_id') for g in data.get('groups', [])]}\"\n )\n models = nous_groups[0][\"models\"]\n assert len(models) >= 20, (\n f\"Live-fetched Nous catalog should expose >=20 entries, got \"\n f\"{len(models)}. The dispatch branch fell through to the four-entry \"\n f\"static list — pre-#1538 behaviour.\"\n )\n finally:\n restore()\n\n def test_nous_model_ids_carry_at_nous_prefix(self, monkeypatch, tmp_path):\n _scrub_provider_env(monkeypatch)\n _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS)\n monkeypatch.setattr(profiles, \"get_active_hermes_home\", lambda: tmp_path)\n\n restore = _swap_in_test_config({\"model\": {\"provider\": \"nous\"}})\n try:\n data = config.get_available_models()\n nous_group = next(g for g in data[\"groups\"] if g[\"provider_id\"] == \"nous\")\n for m in nous_group[\"models\"]:\n assert m[\"id\"].startswith(\"@nous:\"), (\n f\"Every Nous model id must start with '@nous:' so \"\n f\"resolve_model_provider routes through the explicit-provider-hint \"\n f\"branch (matches the static-list invariant from \"\n f\"tests/test_nous_portal_routing.py). Got: {m['id']!r}\"\n )\n finally:\n restore()\n\n def test_nous_labels_carry_via_nous_suffix(self, monkeypatch, tmp_path):\n _scrub_provider_env(monkeypatch)\n _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS)\n monkeypatch.setattr(profiles, \"get_active_hermes_home\", lambda: tmp_path)\n\n restore = _swap_in_test_config({\"model\": {\"provider\": \"nous\"}})\n try:\n data = config.get_available_models()\n nous_group = next(g for g in data[\"groups\"] if g[\"provider_id\"] == \"nous\")\n for m in nous_group[\"models\"]:\n assert m[\"label\"].endswith(\" (via Nous)\"), (\n f\"Every Nous live-fetched label must end with ' (via Nous)' so \"\n f\"the user can distinguish them from same-named direct-provider \"\n f\"entries (e.g. 'Claude Opus 4.7' via direct Anthropic). \"\n f\"Got: {m['label']!r}\"\n )\n finally:\n restore()\n\n def test_nous_live_catalog_includes_recent_models(self, monkeypatch, tmp_path):\n \"\"\"Sanity: the recent-flagship models from the user's bug report\n (Claude Opus 4.7, GPT-5.5, Kimi K2.6) must reach the dropdown.\"\"\"\n _scrub_provider_env(monkeypatch)\n _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS)\n monkeypatch.setattr(profiles, \"get_active_hermes_home\", lambda: tmp_path)\n\n restore = _swap_in_test_config({\"model\": {\"provider\": \"nous\"}})\n try:\n data = config.get_available_models()\n nous_group = next(g for g in data[\"groups\"] if g[\"provider_id\"] == \"nous\")\n ids = {m[\"id\"] for m in nous_group[\"models\"]}\n for required in (\n \"@nous:anthropic/claude-opus-4.7\",\n \"@nous:openai/gpt-5.5\",\n \"@nous:moonshotai/kimi-k2.6\",\n \"@nous:google/gemini-3.1-pro-preview\",\n \"@nous:minimax/minimax-m2.7\",\n ):\n assert required in ids, (\n f\"{required} missing from live-fetched Nous catalog. Either \"\n f\"the hermes_cli dispatch is broken or the @nous: prefix is \"\n f\"missing.\"\n )\n finally:\n restore()\n\n\nclass TestNousStaticFallback:\n \"\"\"When ``hermes_cli`` is not importable or its lookup raises, we fall back\n to the curated four-entry static list — never empty.\"\"\"\n\n def test_static_fallback_when_hermes_cli_raises(self, monkeypatch, tmp_path):\n _scrub_provider_env(monkeypatch)\n _install_fake_hermes_cli(monkeypatch, raise_on_lookup=True)\n monkeypatch.setattr(profiles, \"get_active_hermes_home\", lambda: tmp_path)\n\n restore = _swap_in_test_config({\"model\": {\"provider\": \"nous\"}})\n try:\n data = config.get_available_models()\n nous_groups = [g for g in data.get(\"groups\", []) if g.get(\"provider_id\") == \"nous\"]\n assert nous_groups, (\n \"Nous group must still appear when hermes_cli fails — the \"\n \"branch should fall back to the curated static list.\"\n )\n models = nous_groups[0][\"models\"]\n assert len(models) == 4, (\n f\"Static fallback should expose exactly the four curated entries \"\n f\"in _PROVIDER_MODELS['nous']. Got {len(models)}: \"\n f\"{[m['id'] for m in models]}\"\n )\n for m in models:\n assert m[\"id\"].startswith(\"@nous:\"), m[\"id\"]\n finally:\n restore()\n\n\nclass TestFormatNousLabel:\n \"\"\"Unit tests for the label formatter helper.\"\"\"\n\n def test_strips_vendor_namespace(self):\n from api.config import _format_nous_label\n assert _format_nous_label(\"anthropic/claude-opus-4.7\") == \"Claude Opus 4.7 (via Nous)\"\n assert _format_nous_label(\"openai/gpt-5.4-mini\") == \"GPT 5.4 Mini (via Nous)\"\n\n def test_handles_missing_vendor(self):\n from api.config import _format_nous_label\n # Defensive: id without slash should still render a sane label.\n assert _format_nous_label(\"kimi-k2.6\") == \"Kimi K2.6 (via Nous)\"\n\n def test_handles_variant_after_colon(self):\n from api.config import _format_nous_label\n # Variant rendered in parentheses, mirroring _format_ollama_label.\n out = _format_nous_label(\"minimax/minimax-m2.5:free\")\n assert out.endswith(\" (via Nous)\")\n assert \"Free\" in out\n assert \"MiniMax M2.5\" in out\n\n def test_minimax_renders_mixed_case(self):\n from api.config import _format_nous_label\n # Live wire returns lowercase 'minimax/minimax-...' but the curated\n # convention is mixed-case 'MiniMax'.\n assert _format_nous_label(\"minimax/minimax-m2.7\").startswith(\"MiniMax M2.7\")\n\n def test_label_always_ends_with_via_nous_suffix(self):\n from api.config import _format_nous_label\n for sample in [\n \"anthropic/claude-opus-4.7\",\n \"openai/gpt-5.5\",\n \"google/gemini-3.1-pro-preview\",\n \"moonshotai/kimi-k2.6\",\n \"z-ai/glm-5.1\",\n \"stepfun/step-3.5-flash\",\n ]:\n assert _format_nous_label(sample).endswith(\" (via Nous)\"), sample\n\n\nclass TestStaticListPreservedAsFallback:\n \"\"\"The curated ``_PROVIDER_MODELS['nous']`` entry stays as the static\n fallback; existing routing invariants from\n :mod:`tests.test_nous_portal_routing` must remain valid.\"\"\"\n\n def test_static_list_present(self):\n from api.config import _PROVIDER_MODELS\n assert _PROVIDER_MODELS.get(\"nous\"), (\n \"The curated static Nous list must remain in _PROVIDER_MODELS as \"\n \"a fallback for environments where hermes_cli is unavailable.\"\n )\n\n def test_static_list_keeps_at_nous_prefix(self):\n # Keep parity with tests/test_nous_portal_routing.py — ensures the\n # static fallback path produces correctly-routable ids when used.\n from api.config import _PROVIDER_MODELS\n for m in _PROVIDER_MODELS[\"nous\"]:\n assert m[\"id\"].startswith(\"@nous:\"), m[\"id\"]\n"}}}} |