1 line
101 KiB
JSON
1 line
101 KiB
JSON
{"repo": "2k304/pewdipie-ai", "files": {"src/model_context.py": {"9c1affe9f5636573a4c36e684cffc973f64f3551": {"t": 1785746497.3150768, "n": 13027, "h": "38ce3e2b22876cc8", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nimport sys\nfrom typing import Dict, List, Optional, Tuple\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _normalize_base_for_compare(url: str) -> str:\n url = (url or \"\").strip().rstrip(\"/\")\n for suffix in (\"/chat/completions\", \"/models\", \"/completions\", \"/v1/messages\"):\n if url.endswith(suffix):\n url = url[: -len(suffix)].rstrip(\"/\")\n return url\n\n\ndef _configured_endpoint_kind(url: str) -> Optional[str]:\n \"\"\"Return configured endpoint kind for a chat/base URL when available.\"\"\"\n target = _normalize_base_for_compare(url)\n if not target:\n return None\n if \"core.database\" not in sys.modules:\n return None\n try:\n from core.database import SessionLocal, ModelEndpoint\n db = SessionLocal()\n try:\n rows = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()\n for ep in rows:\n base = _normalize_base_for_compare(getattr(ep, \"base_url\", \"\") or \"\")\n if not base:\n continue\n if target != base and not target.startswith(base + \"/\"):\n continue\n kind = (getattr(ep, \"endpoint_kind\", None) or \"auto\").strip().lower()\n if kind in (\"local\", \"api\", \"proxy\"):\n return kind\n if getattr(ep, \"api_key\", None):\n parsed = urlparse(base)\n host = (parsed.hostname or \"\").lower()\n path = (parsed.path or \"\").rstrip(\"/\")\n if parsed.port != 11434 and \"ollama\" not in host and (path.endswith(\"/v1\") or \"/openai\" in path):\n return \"proxy\"\n return \"auto\"\n finally:\n db.close()\n except Exception:\n return None\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n kind = _configured_endpoint_kind(url)\n if kind in (\"api\", \"proxy\"):\n return False\n if kind == \"local\":\n return True\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-4': 262144,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[Tuple[str, str], int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per (endpoint, model).\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n configured_kind = _configured_endpoint_kind(endpoint_url)\n is_local = _is_local_endpoint(endpoint_url)\n # Key on (endpoint_url, model): the same model id can be served by two\n # different remote endpoints with different real context windows (e.g. a\n # capped proxy vs. the full provider), so caching by model id alone would\n # serve one endpoint's window for the other (issue #2603).\n cache_key = (endpoint_url, model)\n if not is_local and cache_key in _context_cache:\n return _context_cache[cache_key]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request.\n # Local endpoints can restart with a different --max-model-len while keeping\n # the same model id, so always re-query them instead of serving stale cache.\n if not is_local and (ctx != DEFAULT_CONTEXT or configured_kind in (\"api\", \"proxy\")):\n _context_cache[cache_key] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\n\n Picks the LONGEST matching key so a short key never shadows a more specific\n one. Without this, 'o1' (200k) precedes 'o1-mini' (128k) in the table and a\n first-match return would report o1-mini's window as 200k.\n \"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n best_key: Optional[str] = None\n best_ctx: Optional[int] = None\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n if best_key is None or len(key) > len(best_key):\n best_key, best_ctx = key, ctx\n return best_ctx\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n configured_kind = _configured_endpoint_kind(endpoint_url)\n\n # Large OpenAI-compatible proxies can make /models expensive. If the\n # endpoint is explicitly configured as API/proxy, prefer known context\n # metadata (or the default) over downloading the full catalog.\n if configured_kind in (\"api\", \"proxy\"):\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n return DEFAULT_CONTEXT\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n # GitHub Copilot's /models requires auth + X-GitHub-Api-Version headers that\n # aren't available here; an unauthenticated probe just 400s. All Copilot\n # picker models are major API models covered by the known-context table, so\n # rely on that instead of a doomed network call.\n from src.copilot import is_copilot_base\n if is_copilot_base(endpoint_url):\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known or DEFAULT_CONTEXT\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "19a3fc59c9d11e81cccde8a9f44ec010c8214c12": {"t": 1785764968.1364062, "n": 13027, "h": "38ce3e2b22876cc8", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nimport sys\nfrom typing import Dict, List, Optional, Tuple\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _normalize_base_for_compare(url: str) -> str:\n url = (url or \"\").strip().rstrip(\"/\")\n for suffix in (\"/chat/completions\", \"/models\", \"/completions\", \"/v1/messages\"):\n if url.endswith(suffix):\n url = url[: -len(suffix)].rstrip(\"/\")\n return url\n\n\ndef _configured_endpoint_kind(url: str) -> Optional[str]:\n \"\"\"Return configured endpoint kind for a chat/base URL when available.\"\"\"\n target = _normalize_base_for_compare(url)\n if not target:\n return None\n if \"core.database\" not in sys.modules:\n return None\n try:\n from core.database import SessionLocal, ModelEndpoint\n db = SessionLocal()\n try:\n rows = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()\n for ep in rows:\n base = _normalize_base_for_compare(getattr(ep, \"base_url\", \"\") or \"\")\n if not base:\n continue\n if target != base and not target.startswith(base + \"/\"):\n continue\n kind = (getattr(ep, \"endpoint_kind\", None) or \"auto\").strip().lower()\n if kind in (\"local\", \"api\", \"proxy\"):\n return kind\n if getattr(ep, \"api_key\", None):\n parsed = urlparse(base)\n host = (parsed.hostname or \"\").lower()\n path = (parsed.path or \"\").rstrip(\"/\")\n if parsed.port != 11434 and \"ollama\" not in host and (path.endswith(\"/v1\") or \"/openai\" in path):\n return \"proxy\"\n return \"auto\"\n finally:\n db.close()\n except Exception:\n return None\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n kind = _configured_endpoint_kind(url)\n if kind in (\"api\", \"proxy\"):\n return False\n if kind == \"local\":\n return True\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-4': 262144,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[Tuple[str, str], int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per (endpoint, model).\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n configured_kind = _configured_endpoint_kind(endpoint_url)\n is_local = _is_local_endpoint(endpoint_url)\n # Key on (endpoint_url, model): the same model id can be served by two\n # different remote endpoints with different real context windows (e.g. a\n # capped proxy vs. the full provider), so caching by model id alone would\n # serve one endpoint's window for the other (issue #2603).\n cache_key = (endpoint_url, model)\n if not is_local and cache_key in _context_cache:\n return _context_cache[cache_key]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request.\n # Local endpoints can restart with a different --max-model-len while keeping\n # the same model id, so always re-query them instead of serving stale cache.\n if not is_local and (ctx != DEFAULT_CONTEXT or configured_kind in (\"api\", \"proxy\")):\n _context_cache[cache_key] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\n\n Picks the LONGEST matching key so a short key never shadows a more specific\n one. Without this, 'o1' (200k) precedes 'o1-mini' (128k) in the table and a\n first-match return would report o1-mini's window as 200k.\n \"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n best_key: Optional[str] = None\n best_ctx: Optional[int] = None\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n if best_key is None or len(key) > len(best_key):\n best_key, best_ctx = key, ctx\n return best_ctx\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n configured_kind = _configured_endpoint_kind(endpoint_url)\n\n # Large OpenAI-compatible proxies can make /models expensive. If the\n # endpoint is explicitly configured as API/proxy, prefer known context\n # metadata (or the default) over downloading the full catalog.\n if configured_kind in (\"api\", \"proxy\"):\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n return DEFAULT_CONTEXT\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n # GitHub Copilot's /models requires auth + X-GitHub-Api-Version headers that\n # aren't available here; an unauthenticated probe just 400s. All Copilot\n # picker models are major API models covered by the known-context table, so\n # rely on that instead of a doomed network call.\n from src.copilot import is_copilot_base\n if is_copilot_base(endpoint_url):\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known or DEFAULT_CONTEXT\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "1cd0aa2b8c715ec7ad5643531858699ff7f71883": {"t": 1785764969.8635979, "n": 12656, "h": "672578629b092e8b", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nimport sys\nfrom typing import Dict, List, Optional\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _normalize_base_for_compare(url: str) -> str:\n url = (url or \"\").strip().rstrip(\"/\")\n for suffix in (\"/chat/completions\", \"/models\", \"/completions\", \"/v1/messages\"):\n if url.endswith(suffix):\n url = url[: -len(suffix)].rstrip(\"/\")\n return url\n\n\ndef _configured_endpoint_kind(url: str) -> Optional[str]:\n \"\"\"Return configured endpoint kind for a chat/base URL when available.\"\"\"\n target = _normalize_base_for_compare(url)\n if not target:\n return None\n if \"core.database\" not in sys.modules:\n return None\n try:\n from core.database import SessionLocal, ModelEndpoint\n db = SessionLocal()\n try:\n rows = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()\n for ep in rows:\n base = _normalize_base_for_compare(getattr(ep, \"base_url\", \"\") or \"\")\n if not base:\n continue\n if target != base and not target.startswith(base + \"/\"):\n continue\n kind = (getattr(ep, \"endpoint_kind\", None) or \"auto\").strip().lower()\n if kind in (\"local\", \"api\", \"proxy\"):\n return kind\n if getattr(ep, \"api_key\", None):\n parsed = urlparse(base)\n host = (parsed.hostname or \"\").lower()\n path = (parsed.path or \"\").rstrip(\"/\")\n if parsed.port != 11434 and \"ollama\" not in host and (path.endswith(\"/v1\") or \"/openai\" in path):\n return \"proxy\"\n return \"auto\"\n finally:\n db.close()\n except Exception:\n return None\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n kind = _configured_endpoint_kind(url)\n if kind in (\"api\", \"proxy\"):\n return False\n if kind == \"local\":\n return True\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-4': 262144,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[str, int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per model ID.\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n configured_kind = _configured_endpoint_kind(endpoint_url)\n is_local = _is_local_endpoint(endpoint_url)\n if not is_local and model in _context_cache:\n return _context_cache[model]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request.\n # Local endpoints can restart with a different --max-model-len while keeping\n # the same model id, so always re-query them instead of serving stale cache.\n if not is_local and (ctx != DEFAULT_CONTEXT or configured_kind in (\"api\", \"proxy\")):\n _context_cache[model] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\n\n Picks the LONGEST matching key so a short key never shadows a more specific\n one. Without this, 'o1' (200k) precedes 'o1-mini' (128k) in the table and a\n first-match return would report o1-mini's window as 200k.\n \"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n best_key: Optional[str] = None\n best_ctx: Optional[int] = None\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n if best_key is None or len(key) > len(best_key):\n best_key, best_ctx = key, ctx\n return best_ctx\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n configured_kind = _configured_endpoint_kind(endpoint_url)\n\n # Large OpenAI-compatible proxies can make /models expensive. If the\n # endpoint is explicitly configured as API/proxy, prefer known context\n # metadata (or the default) over downloading the full catalog.\n if configured_kind in (\"api\", \"proxy\"):\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n return DEFAULT_CONTEXT\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n # GitHub Copilot's /models requires auth + X-GitHub-Api-Version headers that\n # aren't available here; an unauthenticated probe just 400s. All Copilot\n # picker models are major API models covered by the known-context table, so\n # rely on that instead of a doomed network call.\n from src.copilot import is_copilot_base\n if is_copilot_base(endpoint_url):\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known or DEFAULT_CONTEXT\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "a2e691da2bc5788db9b20dfbc43b4a47c0599057": {"t": 1785764971.8325078, "n": 12148, "h": "4c5781c402b6327f", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nimport sys\nfrom typing import Dict, List, Optional\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _normalize_base_for_compare(url: str) -> str:\n url = (url or \"\").strip().rstrip(\"/\")\n for suffix in (\"/chat/completions\", \"/models\", \"/completions\", \"/v1/messages\"):\n if url.endswith(suffix):\n url = url[: -len(suffix)].rstrip(\"/\")\n return url\n\n\ndef _configured_endpoint_kind(url: str) -> Optional[str]:\n \"\"\"Return configured endpoint kind for a chat/base URL when available.\"\"\"\n target = _normalize_base_for_compare(url)\n if not target:\n return None\n if \"core.database\" not in sys.modules:\n return None\n try:\n from core.database import SessionLocal, ModelEndpoint\n db = SessionLocal()\n try:\n rows = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()\n for ep in rows:\n base = _normalize_base_for_compare(getattr(ep, \"base_url\", \"\") or \"\")\n if not base:\n continue\n if target != base and not target.startswith(base + \"/\"):\n continue\n kind = (getattr(ep, \"endpoint_kind\", None) or \"auto\").strip().lower()\n if kind in (\"local\", \"api\", \"proxy\"):\n return kind\n if getattr(ep, \"api_key\", None):\n parsed = urlparse(base)\n host = (parsed.hostname or \"\").lower()\n path = (parsed.path or \"\").rstrip(\"/\")\n if parsed.port != 11434 and \"ollama\" not in host and (path.endswith(\"/v1\") or \"/openai\" in path):\n return \"proxy\"\n return \"auto\"\n finally:\n db.close()\n except Exception:\n return None\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n kind = _configured_endpoint_kind(url)\n if kind in (\"api\", \"proxy\"):\n return False\n if kind == \"local\":\n return True\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-4': 262144,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[str, int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per model ID.\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n configured_kind = _configured_endpoint_kind(endpoint_url)\n is_local = _is_local_endpoint(endpoint_url)\n if not is_local and model in _context_cache:\n return _context_cache[model]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request.\n # Local endpoints can restart with a different --max-model-len while keeping\n # the same model id, so always re-query them instead of serving stale cache.\n if not is_local and (ctx != DEFAULT_CONTEXT or configured_kind in (\"api\", \"proxy\")):\n _context_cache[model] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\n\n Picks the LONGEST matching key so a short key never shadows a more specific\n one. Without this, 'o1' (200k) precedes 'o1-mini' (128k) in the table and a\n first-match return would report o1-mini's window as 200k.\n \"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n best_key: Optional[str] = None\n best_ctx: Optional[int] = None\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n if best_key is None or len(key) > len(best_key):\n best_key, best_ctx = key, ctx\n return best_ctx\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n configured_kind = _configured_endpoint_kind(endpoint_url)\n\n # Large OpenAI-compatible proxies can make /models expensive. If the\n # endpoint is explicitly configured as API/proxy, prefer known context\n # metadata (or the default) over downloading the full catalog.\n if configured_kind in (\"api\", \"proxy\"):\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n return DEFAULT_CONTEXT\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "39848a168b79eccfbe6703ff97f6d27c93abea67": {"t": 1785764973.3038447, "n": 9732, "h": "1b669a8137b78b2c", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nfrom typing import Dict, List, Optional\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-4': 262144,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[str, int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per model ID.\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n is_local = _is_local_endpoint(endpoint_url)\n if not is_local and model in _context_cache:\n return _context_cache[model]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request.\n # Local endpoints can restart with a different --max-model-len while keeping\n # the same model id, so always re-query them instead of serving stale cache.\n if not is_local and ctx != DEFAULT_CONTEXT:\n _context_cache[model] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\n\n Picks the LONGEST matching key so a short key never shadows a more specific\n one. Without this, 'o1' (200k) precedes 'o1-mini' (128k) in the table and a\n first-match return would report o1-mini's window as 200k.\n \"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n best_key: Optional[str] = None\n best_ctx: Optional[int] = None\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n if best_key is None or len(key) > len(best_key):\n best_key, best_ctx = key, ctx\n return best_ctx\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "d06b6d87d3e0993079d17e4e936efae0056e6fdf": {"t": 1785764974.7559612, "n": 9709, "h": "ba3f4d784429f045", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nfrom typing import Dict, List, Optional\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[str, int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per model ID.\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n is_local = _is_local_endpoint(endpoint_url)\n if not is_local and model in _context_cache:\n return _context_cache[model]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request.\n # Local endpoints can restart with a different --max-model-len while keeping\n # the same model id, so always re-query them instead of serving stale cache.\n if not is_local and ctx != DEFAULT_CONTEXT:\n _context_cache[model] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\n\n Picks the LONGEST matching key so a short key never shadows a more specific\n one. Without this, 'o1' (200k) precedes 'o1-mini' (128k) in the table and a\n first-match return would report o1-mini's window as 200k.\n \"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n best_key: Optional[str] = None\n best_ctx: Optional[int] = None\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n if best_key is None or len(key) > len(best_key):\n best_key, best_ctx = key, ctx\n return best_ctx\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "cd6041477c611dc22ca60ed983ef440fe37cc170": {"t": 1785764976.340516, "n": 9323, "h": "44ae3812d629e388", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nfrom typing import Dict, List, Optional\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[str, int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per model ID.\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n is_local = _is_local_endpoint(endpoint_url)\n if not is_local and model in _context_cache:\n return _context_cache[model]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request.\n # Local endpoints can restart with a different --max-model-len while keeping\n # the same model id, so always re-query them instead of serving stale cache.\n if not is_local and ctx != DEFAULT_CONTEXT:\n _context_cache[model] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n return ctx\n return None\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "d885c7046281e3294c00f744c60f45a0854b1666": {"t": 1785764978.509006, "n": 9078, "h": "7292581e34975b49", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nfrom typing import Dict, List, Optional\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\", \"host.docker.internal\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[str, int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per model ID.\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n if model in _context_cache:\n return _context_cache[model]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request\n if ctx != DEFAULT_CONTEXT:\n _context_cache[model] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n return ctx\n return None\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}, "e5c99a5eee62fc7df0e2dca5e46cbc87399047d6": {"t": 1785764980.3207452, "n": 9054, "h": "a45047d64b00a429", "c": "\"\"\"\nmodel_context.py\n\nQuery and cache model context window sizes from OpenAI-compatible APIs.\nProvides token estimation for context usage tracking.\n\"\"\"\n\nimport logging\nfrom typing import Dict, List, Optional\n\nfrom urllib.parse import urlparse\n\nimport httpx\n\nlogger = logging.getLogger(__name__)\n\n_LOCAL_HOSTS = {\"localhost\", \"127.0.0.1\", \"0.0.0.0\", \"::1\"}\n_PRIVATE_PREFIXES = (\"10.\", \"172.16.\", \"172.17.\", \"172.18.\", \"172.19.\",\n \"172.20.\", \"172.21.\", \"172.22.\", \"172.23.\", \"172.24.\",\n \"172.25.\", \"172.26.\", \"172.27.\", \"172.28.\", \"172.29.\",\n \"172.30.\", \"172.31.\", \"192.168.\", \"100.\")\n\n\ndef _is_local_endpoint(url: str) -> bool:\n \"\"\"Check if URL points to a local/private/tailscale address.\"\"\"\n try:\n host = urlparse(url).hostname or \"\"\n return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES)\n except Exception:\n return False\n\n# ---------------------------------------------------------------------------\n# Constants\n# ---------------------------------------------------------------------------\nDEFAULT_CONTEXT = 128000\nREQUEST_TIMEOUT = 5\n\n# Known context windows for major API models (used as fallback when /models\n# endpoint doesn't report context_length).\n# Substring matching — use the shortest unique prefix so variants get caught.\nKNOWN_CONTEXT_WINDOWS = {\n # --- Anthropic ---\n 'claude-sonnet-4-5': 200000,\n 'claude-sonnet-4-6': 200000,\n 'claude-sonnet-4': 200000,\n 'claude-opus-4': 200000,\n 'claude-haiku-4': 200000,\n 'claude-haiku-3-5': 200000,\n 'claude-3-5-sonnet': 200000,\n 'claude-3-5-haiku': 200000,\n 'claude-3-opus': 200000,\n 'claude-3-sonnet': 200000,\n 'claude-3-haiku': 200000,\n\n # --- OpenAI ---\n 'gpt-5': 400000,\n 'gpt-4.1': 1047576,\n 'gpt-4.1-mini': 1047576,\n 'gpt-4.1-nano': 1047576,\n 'gpt-4o': 128000,\n 'gpt-4o-mini': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4': 8192,\n 'gpt-3.5-turbo': 16385,\n 'o1': 200000,\n 'o1-mini': 128000,\n 'o1-pro': 200000,\n 'o3': 200000,\n 'o3-mini': 200000,\n 'o4-mini': 200000,\n\n # --- DeepSeek ---\n 'deepseek-chat': 64000,\n 'deepseek-coder': 64000,\n 'deepseek-reasoner': 64000,\n 'deepseek-r1': 64000,\n 'deepseek-v3': 64000,\n 'deepseek-v2': 64000,\n\n # --- Google ---\n 'gemini-2.5-pro': 1048576,\n 'gemini-2.5-flash': 1048576,\n 'gemini-2.0-flash': 1048576,\n 'gemini-1.5-pro': 1048576,\n 'gemini-1.5-flash': 1048576,\n 'gemma-3': 128000,\n 'gemma-2': 8192,\n\n # --- Mistral ---\n 'mistral-large': 128000,\n 'mistral-medium': 32000,\n 'mistral-small': 32000,\n 'mistral-nemo': 128000,\n 'mistral-7b': 32000,\n 'mixtral': 32000,\n 'codestral': 32000,\n 'pixtral': 128000,\n\n # --- xAI ---\n 'grok-4': 131072,\n 'grok-3': 131072,\n 'grok-2': 131072,\n\n # --- Meta / Llama ---\n 'llama-4': 1048576,\n 'llama-3.3': 131072,\n 'llama-3.2': 131072,\n 'llama-3.1': 131072,\n 'llama-3': 131072,\n\n # --- Qwen ---\n 'qwen3': 131072,\n 'qwen2.5': 131072,\n 'qwen2': 32768,\n 'qwq': 32768,\n\n # --- Cohere ---\n 'command-r-plus': 128000,\n 'command-r': 128000,\n 'command-a': 256000,\n\n # --- Perplexity ---\n 'sonar-pro': 200000,\n 'sonar': 128000,\n\n # --- MiniMax ---\n 'minimax': 1000000,\n\n # --- Moonshot / Kimi ---\n 'moonshot': 128000,\n 'kimi': 128000,\n\n # --- Microsoft ---\n 'phi-4': 16000,\n 'phi-3': 128000,\n\n # --- Nvidia ---\n 'nemotron': 131072,\n\n # --- Yi ---\n 'yi-large': 32768,\n 'yi-1.5': 16384,\n\n # --- 01.ai ---\n 'yi-lightning': 16384,\n\n # --- Nous ---\n 'hermes': 131072,\n 'nous-hermes': 131072,\n\n # --- Open community ---\n 'dolphin': 32768,\n 'mythomax': 4096,\n 'wizard': 32768,\n 'openchat': 8192,\n 'solar': 32768,\n}\n\n# ---------------------------------------------------------------------------\n# Cache\n# ---------------------------------------------------------------------------\n_context_cache: Dict[str, int] = {}\n\n\ndef get_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Get the context window size for a model.\n\n Queries /v1/models on the endpoint and looks for context_length\n or context_window fields. Caches result per model ID.\n Falls back to DEFAULT_CONTEXT if unavailable.\n \"\"\"\n if model in _context_cache:\n return _context_cache[model]\n\n ctx = _query_context_length(endpoint_url, model)\n # Only cache non-default values to allow retry on next request\n if ctx != DEFAULT_CONTEXT:\n _context_cache[model] = ctx\n logger.info(f\"Context length for {model}: {ctx}\")\n return ctx\n\n\ndef _lookup_known(model: str) -> Optional[int]:\n \"\"\"Check known context windows by substring match.\"\"\"\n name = model.lower()\n basename = name.split(\"/\")[-1] if \"/\" in name else name\n basename = basename.split(\":\")[0] # strip :free, :extended etc.\n for key, ctx in KNOWN_CONTEXT_WINDOWS.items():\n if key in basename or key in name:\n return ctx\n return None\n\n\ndef _query_context_length(endpoint_url: str, model: str) -> int:\n \"\"\"Query the model API for context length.\"\"\"\n known = _lookup_known(model)\n api_ctx = None\n\n # Try llama.cpp /slots endpoint first — reports actual serving context\n if _is_local_endpoint(endpoint_url):\n try:\n base = endpoint_url.split(\"/v1\")[0] if \"/v1\" in endpoint_url else endpoint_url.rsplit(\"/\", 1)[0]\n r = httpx.get(f\"{base}/slots\", timeout=REQUEST_TIMEOUT)\n if r.is_success:\n slots = r.json()\n if isinstance(slots, list) and slots:\n n_ctx = slots[0].get(\"n_ctx\")\n if n_ctx and isinstance(n_ctx, int) and n_ctx > 0:\n logger.info(f\"llama.cpp /slots reports n_ctx={n_ctx} for {model}\")\n return n_ctx\n except Exception:\n pass\n\n models_url = endpoint_url.replace(\"/chat/completions\", \"/models\")\n try:\n r = httpx.get(models_url, timeout=REQUEST_TIMEOUT)\n if r.is_success:\n data = r.json()\n models_list = data.get(\"data\") or []\n\n for m in models_list:\n mid = m.get(\"id\", \"\")\n if mid == model or mid.split(\"/\")[-1] == model.split(\"/\")[-1]:\n for field in (\n \"context_length\",\n \"context_window\",\n \"max_model_len\",\n \"max_context_length\",\n \"max_seq_len\",\n ):\n val = m.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n\n if not api_ctx:\n meta = m.get(\"meta\") or m.get(\"model_extra\") or {}\n if isinstance(meta, dict):\n # n_ctx is the actual serving context (set via -c flag in llama.cpp)\n for field in (\"n_ctx\", \"context_length\", \"context_window\", \"max_model_len\"):\n val = meta.get(field)\n if val and isinstance(val, (int, float)) and val > 0:\n api_ctx = int(val)\n break\n break\n except Exception as e:\n logger.debug(f\"Failed to query context length for {model}: {e}\")\n\n # For local/self-hosted endpoints, trust the API value (user set --max-model-len)\n # For cloud APIs, use the larger value (API can report low defaults)\n if api_ctx and known:\n _is_local = _is_local_endpoint(endpoint_url)\n if _is_local and api_ctx < known:\n logger.info(f\"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value\")\n return api_ctx\n result = max(api_ctx, known)\n if api_ctx < known:\n logger.info(f\"API reported {api_ctx} for {model}, using known {known} instead\")\n return result\n if api_ctx:\n return api_ctx\n if known:\n logger.info(f\"Using known context window for {model}: {known}\")\n return known\n\n return DEFAULT_CONTEXT\n\n\ndef estimate_tokens(messages: List[Dict]) -> int:\n \"\"\"Rough token estimate for a list of messages.\n\n Uses chars * 0.3 which is closer to real BPE tokenizer output\n than the commonly-cited chars/4 (which underestimates by ~20-30%).\n Also adds ~4 tokens per message for role/formatting overhead.\n \"\"\"\n total = 0\n for msg in messages:\n total += 4 # per-message overhead (role, separators)\n content = msg.get(\"content\", \"\")\n if isinstance(content, str):\n total += int(len(content) * 0.3)\n elif isinstance(content, list):\n for item in content:\n if isinstance(item, dict) and item.get(\"type\") == \"text\":\n total += int(len(item.get(\"text\", \"\")) * 0.3)\n return total\n"}}}} |