1 line
3.2 MiB
Plaintext
1 line
3.2 MiB
Plaintext
{"repo": "aden-hive/hive", "files": {"core/framework/llm/model_catalog.json": {"3c390c9e6ca3a2b141567bea986079fedbcd7db1": {"t": 1785674222.0070014, "n": 14678, "h": "0d4dfbe29fff56e2", "c": "{\n \"schema_version\": 1,\n \"providers\": {\n \"anthropic\": {\n \"default_model\": \"claude-haiku-4-5-20251001\",\n \"models\": [\n {\n \"id\": \"claude-haiku-4-5-20251001\",\n \"label\": \"Haiku 4.5 - Fast + cheap\",\n \"recommended\": false,\n \"max_tokens\": 64000,\n \"max_context_tokens\": 136000,\n \"supports_vision\": true\n },\n {\n \"id\": \"claude-sonnet-4-5-20250929\",\n \"label\": \"Sonnet 4.5 - Best balance\",\n \"recommended\": false,\n \"max_tokens\": 64000,\n \"max_context_tokens\": 136000,\n \"supports_vision\": true\n },\n {\n \"id\": \"claude-opus-4-6\",\n \"label\": \"Opus 4.6 - Most capable\",\n \"recommended\": true,\n \"max_tokens\": 128000,\n \"max_context_tokens\": 872000,\n \"supports_vision\": true\n }\n ]\n },\n \"openai\": {\n \"default_model\": \"gpt-5.5\",\n \"models\": [\n {\n \"id\": \"gpt-5.5\",\n \"label\": \"GPT-5.5 - Frontier coding + reasoning\",\n \"recommended\": true,\n \"max_tokens\": 128000,\n \"max_context_tokens\": 1050000,\n \"pricing_usd_per_mtok\": {\n \"input\": 5.00,\n \"output\": 30.00\n },\n \"supports_vision\": true\n },\n {\n \"id\": \"gpt-5.4\",\n \"label\": \"GPT-5.4 - Previous flagship\",\n \"recommended\": false,\n \"max_tokens\": 128000,\n \"max_context_tokens\": 960000,\n \"supports_vision\": true\n },\n {\n \"id\": \"gpt-5.4-mini\",\n \"label\": \"GPT-5.4 Mini - Faster + cheaper\",\n \"recommended\": false,\n \"max_tokens\": 128000,\n \"max_context_tokens\": 400000,\n \"supports_vision\": true\n },\n {\n \"id\": \"gpt-5.4-nano\",\n \"label\": \"GPT-5.4 Nano - Cheapest high-volume\",\n \"recommended\": false,\n \"max_tokens\": 128000,\n \"max_context_tokens\": 400000,\n \"supports_vision\": true\n }\n ]\n },\n \"gemini\": {\n \"default_model\": \"gemini-3-flash-preview\",\n \"models\": [\n {\n \"id\": \"gemini-3-flash-preview\",\n \"label\": \"Gemini 3 Flash - Fast\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 240000,\n \"supports_vision\": true\n },\n {\n \"id\": \"gemini-3.1-pro-preview-customtools\",\n \"label\": \"Gemini 3.1 Pro - Best quality\",\n \"recommended\": true,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 240000,\n \"supports_vision\": true\n }\n ]\n },\n \"groq\": {\n \"default_model\": \"openai/gpt-oss-120b\",\n \"models\": [\n {\n \"id\": \"openai/gpt-oss-120b\",\n \"label\": \"GPT-OSS 120B - Best reasoning\",\n \"recommended\": true,\n \"max_tokens\": 65536,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n },\n {\n \"id\": \"openai/gpt-oss-20b\",\n \"label\": \"GPT-OSS 20B - Fast + cheaper\",\n \"recommended\": false,\n \"max_tokens\": 65536,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n },\n {\n \"id\": \"llama-3.3-70b-versatile\",\n \"label\": \"Llama 3.3 70B - General purpose\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n },\n {\n \"id\": \"llama-3.1-8b-instant\",\n \"label\": \"Llama 3.1 8B - Fastest\",\n \"recommended\": false,\n \"max_tokens\": 131072,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n }\n ]\n },\n \"cerebras\": {\n \"default_model\": \"gpt-oss-120b\",\n \"models\": [\n {\n \"id\": \"gpt-oss-120b\",\n \"label\": \"GPT-OSS 120B - Best production reasoning\",\n \"recommended\": true,\n \"max_tokens\": 40960,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n },\n {\n \"id\": \"zai-glm-4.7\",\n \"label\": \"Z.ai GLM 4.7 - Strong coding preview\",\n \"recommended\": true,\n \"max_tokens\": 40960,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n },\n {\n \"id\": \"qwen-3-235b-a22b-instruct-2507\",\n \"label\": \"Qwen 3 235B Instruct - Frontier preview\",\n \"recommended\": false,\n \"max_tokens\": 40960,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n }\n ]\n },\n \"minimax\": {\n \"default_model\": \"MiniMax-M2.7\",\n \"models\": [\n {\n \"id\": \"MiniMax-M2.7\",\n \"label\": \"MiniMax M2.7 - Best coding quality\",\n \"recommended\": true,\n \"max_tokens\": 40960,\n \"max_context_tokens\": 180000,\n \"pricing_usd_per_mtok\": {\n \"input\": 0.30,\n \"output\": 1.20\n },\n \"supports_vision\": false\n },\n {\n \"id\": \"MiniMax-M2.5\",\n \"label\": \"MiniMax M2.5 - Strong value\",\n \"recommended\": false,\n \"max_tokens\": 40960,\n \"max_context_tokens\": 180000,\n \"supports_vision\": false\n }\n ]\n },\n \"mistral\": {\n \"default_model\": \"mistral-large-2512\",\n \"models\": [\n {\n \"id\": \"mistral-large-2512\",\n \"label\": \"Mistral Large 3 - Best quality\",\n \"recommended\": true,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 256000,\n \"supports_vision\": true\n },\n {\n \"id\": \"mistral-medium-2508\",\n \"label\": \"Mistral Medium 3.1 - Balanced\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 128000,\n \"supports_vision\": true\n },\n {\n \"id\": \"mistral-small-2603\",\n \"label\": \"Mistral Small 4 - Fast + capable\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 256000,\n \"supports_vision\": true\n },\n {\n \"id\": \"codestral-2508\",\n \"label\": \"Codestral - Coding specialist\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 128000,\n \"supports_vision\": false\n }\n ]\n },\n \"together\": {\n \"default_model\": \"deepseek-ai/DeepSeek-V3.1\",\n \"models\": [\n {\n \"id\": \"deepseek-ai/DeepSeek-V3.1\",\n \"label\": \"DeepSeek V3.1 - Best general coding\",\n \"recommended\": true,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 128000,\n \"supports_vision\": false\n },\n {\n \"id\": \"Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8\",\n \"label\": \"Qwen3 Coder 480B - Advanced coding\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 262144,\n \"supports_vision\": false\n },\n {\n \"id\": \"openai/gpt-oss-120b\",\n \"label\": \"GPT-OSS 120B - Strong reasoning\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 128000,\n \"supports_vision\": false\n },\n {\n \"id\": \"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\n \"label\": \"Llama 3.3 70B Turbo - Fast baseline\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 131072,\n \"supports_vision\": false\n }\n ]\n },\n \"deepseek\": {\n \"default_model\": \"deepseek-v4-pro\",\n \"models\": [\n {\n \"id\": \"deepseek-v4-pro\",\n \"label\": \"DeepSeek V4 Pro - Most capable\",\n \"recommended\": true,\n \"max_tokens\": 384000,\n \"max_context_tokens\": 1000000,\n \"pricing_usd_per_mtok\": {\n \"input\": 1.74,\n \"output\": 3.48,\n \"cache_read\": 0.145\n },\n \"supports_vision\": false\n },\n {\n \"id\": \"deepseek-v4-flash\",\n \"label\": \"DeepSeek V4 Flash - Fast + cheap\",\n \"recommended\": true,\n \"max_tokens\": 384000,\n \"max_context_tokens\": 1000000,\n \"pricing_usd_per_mtok\": {\n \"input\": 0.14,\n \"output\": 0.28,\n \"cache_read\": 0.028\n },\n \"supports_vision\": false\n },\n {\n \"id\": \"deepseek-reasoner\",\n \"label\": \"DeepSeek Reasoner - Legacy (deprecating)\",\n \"recommended\": false,\n \"max_tokens\": 64000,\n \"max_context_tokens\": 128000,\n \"supports_vision\": false\n }\n ]\n },\n \"kimi\": {\n \"default_model\": \"kimi-k2.5\",\n \"models\": [\n {\n \"id\": \"kimi-k2.5\",\n \"label\": \"Kimi K2.5 - Best coding\",\n \"recommended\": true,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 200000,\n \"pricing_usd_per_mtok\": {\n \"input\": 0.60,\n \"output\": 2.50,\n \"cache_read\": 0.15\n },\n \"supports_vision\": true\n }\n ]\n },\n \"hive\": {\n \"default_model\": \"queen\",\n \"models\": [\n {\n \"id\": \"queen\",\n \"label\": \"Queen - Hive native\",\n \"recommended\": true,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 180000,\n \"supports_vision\": false\n },\n {\n \"id\": \"kimi-k2.5\",\n \"label\": \"Kimi 2.5 - Via Hive\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 240000,\n \"supports_vision\": true\n },\n {\n \"id\": \"glm-5.1\",\n \"label\": \"GLM-5.1 - Via Hive\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 180000,\n \"pricing_usd_per_mtok\": {\n \"input\": 1.40,\n \"output\": 4.40,\n \"cache_read\": 0.26,\n \"cache_creation\": 0.0\n },\n \"supports_vision\": false\n }\n ]\n },\n \"openrouter\": {\n \"default_model\": \"openai/gpt-5.4\",\n \"models\": [\n {\n \"id\": \"openai/gpt-5.4\",\n \"label\": \"GPT-5.4 - Best overall\",\n \"recommended\": true,\n \"max_tokens\": 128000,\n \"max_context_tokens\": 872000,\n \"supports_vision\": true\n },\n {\n \"id\": \"anthropic/claude-sonnet-4.6\",\n \"label\": \"Claude Sonnet 4.6 - Best coding balance\",\n \"recommended\": false,\n \"max_tokens\": 64000,\n \"max_context_tokens\": 872000,\n \"supports_vision\": true\n },\n {\n \"id\": \"anthropic/claude-opus-4.6\",\n \"label\": \"Claude Opus 4.6 - Most capable\",\n \"recommended\": false,\n \"max_tokens\": 128000,\n \"max_context_tokens\": 872000,\n \"supports_vision\": true\n },\n {\n \"id\": \"google/gemini-3.1-pro-preview-customtools\",\n \"label\": \"Gemini 3.1 Pro Preview - Long-context reasoning\",\n \"recommended\": false,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 872000,\n \"supports_vision\": true\n },\n {\n \"id\": \"qwen/qwen3.6-plus\",\n \"label\": \"Qwen 3.6 Plus - Strong reasoning\",\n \"recommended\": true,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 240000,\n \"supports_vision\": false\n },\n {\n \"id\": \"z-ai/glm-5v-turbo\",\n \"label\": \"GLM-5V Turbo - Vision capable\",\n \"recommended\": true,\n \"max_tokens\": 32768,\n \"max_context_tokens\": 192000,\n \"supports_vision\": true\n },\n {\n \"id\": \"z-ai/glm-5.1\",\n \"label\": \"GLM-5.1 - Better but Slower\",\n \"recommended\": true,\n \"max_tokens\": 40960,\n \"max_context_tokens\": 192000,\n \"pricing_usd_per_mtok\": {\n \"input\": 1.40,\n \"output\": 4.40,\n \"cache_read\": 0.26,\n \"cache_creation\": 0.0\n },\n \"supports_vision\": false\n },\n {\n \"id\": \"minimax/minimax-m2.7\",\n \"label\": \"Minimax M2.7 - Minimax flagship\",\n \"recommended\": false,\n \"max_tokens\": 40960,\n \"max_context_tokens\": 180000,\n \"pricing_usd_per_mtok\": {\n \"input\": 0.30,\n \"output\": 1.20\n },\n \"supports_vision\": false\n },\n {\n \"id\": \"xiaomi/mimo-v2-pro\",\n \"label\": \"MiMo V2 Pro - Xiaomi multimodal\",\n \"recommended\": true,\n \"max_tokens\": 64000,\n \"max_context_tokens\": 240000,\n \"supports_vision\": true\n }\n ]\n }\n },\n \"presets\": {\n \"claude_code\": {\n \"provider\": \"anthropic\",\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 128000,\n \"max_context_tokens\": 872000\n },\n \"zai_code\": {\n \"provider\": \"openai\",\n \"api_key_env_var\": \"ZAI_API_KEY\",\n \"model\": \"glm-5.1\",\n \"max_tokens\": 32768,\n \"max_context_tokens\": 180000,\n \"api_base\": \"https://api.z.ai/api/coding/paas/v4\"\n },\n \"codex\": {\n \"provider\": \"openai\",\n \"model\": \"gpt-5.3-codex\",\n \"max_tokens\": 16384,\n \"max_context_tokens\": 120000,\n \"api_base\": \"https://chatgpt.com/backend-api/codex\"\n },\n \"minimax_code\": {\n \"provider\": \"minimax\",\n \"api_key_env_var\": \"MINIMAX_API_KEY\",\n \"model\": \"MiniMax-M2.7\",\n \"max_tokens\": 40960,\n \"max_context_tokens\": 180800,\n \"api_base\": \"https://api.minimax.io/v1\"\n },\n \"kimi_code\": {\n \"provider\": \"kimi\",\n \"api_key_env_var\": \"KIMI_API_KEY\",\n \"model\": \"kimi-k2.5\",\n \"max_tokens\": 32768,\n \"max_context_tokens\": 240000,\n \"api_base\": \"https://api.kimi.com/coding\"\n },\n \"hive_llm\": {\n \"provider\": \"hive\",\n \"api_key_env_var\": \"HIVE_API_KEY\",\n \"model\": \"queen\",\n \"max_tokens\": 32768,\n \"max_context_tokens\": 180000,\n \"api_base\": \"https://api.adenhq.com\",\n \"model_choices\": [\n {\n \"id\": \"queen\",\n \"label\": \"queen\",\n \"recommended\": true\n },\n {\n \"id\": \"kimi-k2.5\",\n \"label\": \"kimi-k2.5\",\n \"recommended\": false\n },\n {\n \"id\": \"glm-5.1\",\n \"label\": \"glm-5.1\",\n \"recommended\": false\n }\n ]\n },\n \"antigravity\": {\n \"provider\": \"openai\",\n \"model\": \"gemini-3-flash\",\n \"max_tokens\": 32768,\n \"max_context_tokens\": 1000000\n },\n \"ollama_local\": {\n \"provider\": \"ollama\",\n \"max_tokens\": 8192,\n \"max_context_tokens\": 16384,\n \"api_base\": \"http://localhost:11434\"\n }\n }\n}"}}, "core/framework/llm/litellm.py": {"3c390c9e6ca3a2b141567bea986079fedbcd7db1": {"t": 1785746546.6960933, "n": 123642, "h": "5f78c7bd9b261fd5", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _api_base_needs_bearer_auth(api_base: str | None) -> bool:\n \"\"\"Return True when api_base points at an Anthropic-compatible endpoint\n that authenticates via ``Authorization: Bearer`` rather than ``x-api-key``.\n\n The Hive LLM proxy (Rust service in hive-backend/llm/) speaks the\n Anthropic Messages API but mints user-scoped JWTs and validates them\n via Bearer auth. Default upstream Anthropic endpoints (api.anthropic.com,\n Kimi's api.kimi.com/coding) keep using x-api-key, so the override is\n scoped to known hive-proxy hosts plus the env-configured override.\n \"\"\"\n if not api_base:\n return False\n # Strip protocol, port, and path so a plain hostname compare is enough\n # for the common cases.\n lowered = api_base.lower()\n for host in (\"adenhq.com\", \"open-hive.com\", \"127.0.0.1:8890\", \"localhost:8890\"):\n if host in lowered:\n return True\n override = os.environ.get(\"HIVE_LLM_BASE_URL\")\n if override and override.lower() in lowered:\n return True\n return False\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\ndef rewrite_proxy_model(\n model: str, api_key: str | None, api_base: str | None\n) -> tuple[str, str | None, dict[str, str]]:\n \"\"\"Apply Hive/Kimi proxy rewrites for any caller of ``litellm.acompletion``.\n\n Both the Hive LLM proxy and Kimi For Coding expose Anthropic-API-\n compatible endpoints. LiteLLM doesn't recognise the ``hive/`` or\n ``kimi/`` prefixes natively, so we rewrite them to ``anthropic/``\n here. For the Hive proxy we also stamp a Bearer token into\n ``extra_headers`` because litellm's Anthropic handler only sends\n ``x-api-key`` and the proxy expects ``Authorization: Bearer``.\n\n Used by ad-hoc ``litellm.acompletion`` callers (e.g. the vision-\n fallback subagent in ``caption_tool_image``) so they hit the same\n proxy with the same auth as the main agent's ``LiteLLMProvider``.\n The provider's own ``__init__`` keeps its inlined rewrite for now —\n this helper is the single source of truth for ad-hoc callers.\n\n Returns: (rewritten_model, normalised_api_base, extra_headers).\n The ``extra_headers`` dict is non-empty only for the Hive proxy\n (and only when ``api_key`` is provided).\n \"\"\"\n extra_headers: dict[str, str] = {}\n if model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n # Hive proxy expects Bearer auth; litellm's Anthropic handler\n # only sends x-api-key without this nudge.\n if api_key:\n extra_headers[\"Authorization\"] = f\"Bearer {api_key}\"\n return model, api_base, extra_headers\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n\n# Directory for dumping failed requests. Resolved lazily so HIVE_HOME\n# overrides (set by the desktop shell) take effect even if this module\n# is imported before framework.config picks up the override.\ndef _failed_requests_dir() -> Path:\n from framework.config import HIVE_HOME\n\n return HIVE_HOME / \"failed_requests\"\n\n\n# Maximum number of dump files to retain in $HIVE_HOME/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(model, input_tokens, output_tokens, cache_read, cache_creation)\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens)\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (getattr(_details, \"cache_write_tokens\", 0) or 0 if _details is not None else 0) or (\n getattr(usage, \"cache_creation_input_tokens\", 0) or 0\n )\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n _failed_requests_dir().glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n dump_dir = _failed_requests_dir()\n dump_dir.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = dump_dir / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {_failed_requests_dir()}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n self._hive_proxy_auth = bool(_original_model.lower().startswith(\"hive/\"))\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n self._hive_proxy_auth = bool(_original_model.lower().startswith(\"hive/\"))\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # The Hive LLM proxy fronts GLM, which drifts into \"explain\n # the plan\" mode on long-context turns instead of emitting\n # tool_use blocks (verified 2026-04-28: tool_choice=null →\n # text-only stop=stop; tool_choice=required → clean\n # tool_use). Force a tool call when tools are available\n # so queens can't get stuck in chat mode. Callers that\n # legitimately want a non-tool turn can override via\n # extra_kwargs.\n kwargs.setdefault(\"tool_choice\", \"required\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # See `complete()` for the rationale: GLM behind the Hive\n # proxy needs forcing or it goes chat-mode on long contexts.\n kwargs.setdefault(\"tool_choice\", \"required\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n\n from framework.config import HIVE_HOME as _HIVE_HOME\n\n _debug_dir = _HIVE_HOME / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # See `complete()` for the rationale: GLM behind the Hive\n # proxy needs forcing or it goes chat-mode on long contexts.\n kwargs.setdefault(\"tool_choice\", \"required\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "d9813288d95e1e115d527ae4b5af00a8e36d58e2": {"t": 1785765513.4959064, "n": 123642, "h": "5f78c7bd9b261fd5", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _api_base_needs_bearer_auth(api_base: str | None) -> bool:\n \"\"\"Return True when api_base points at an Anthropic-compatible endpoint\n that authenticates via ``Authorization: Bearer`` rather than ``x-api-key``.\n\n The Hive LLM proxy (Rust service in hive-backend/llm/) speaks the\n Anthropic Messages API but mints user-scoped JWTs and validates them\n via Bearer auth. Default upstream Anthropic endpoints (api.anthropic.com,\n Kimi's api.kimi.com/coding) keep using x-api-key, so the override is\n scoped to known hive-proxy hosts plus the env-configured override.\n \"\"\"\n if not api_base:\n return False\n # Strip protocol, port, and path so a plain hostname compare is enough\n # for the common cases.\n lowered = api_base.lower()\n for host in (\"adenhq.com\", \"open-hive.com\", \"127.0.0.1:8890\", \"localhost:8890\"):\n if host in lowered:\n return True\n override = os.environ.get(\"HIVE_LLM_BASE_URL\")\n if override and override.lower() in lowered:\n return True\n return False\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\ndef rewrite_proxy_model(\n model: str, api_key: str | None, api_base: str | None\n) -> tuple[str, str | None, dict[str, str]]:\n \"\"\"Apply Hive/Kimi proxy rewrites for any caller of ``litellm.acompletion``.\n\n Both the Hive LLM proxy and Kimi For Coding expose Anthropic-API-\n compatible endpoints. LiteLLM doesn't recognise the ``hive/`` or\n ``kimi/`` prefixes natively, so we rewrite them to ``anthropic/``\n here. For the Hive proxy we also stamp a Bearer token into\n ``extra_headers`` because litellm's Anthropic handler only sends\n ``x-api-key`` and the proxy expects ``Authorization: Bearer``.\n\n Used by ad-hoc ``litellm.acompletion`` callers (e.g. the vision-\n fallback subagent in ``caption_tool_image``) so they hit the same\n proxy with the same auth as the main agent's ``LiteLLMProvider``.\n The provider's own ``__init__`` keeps its inlined rewrite for now —\n this helper is the single source of truth for ad-hoc callers.\n\n Returns: (rewritten_model, normalised_api_base, extra_headers).\n The ``extra_headers`` dict is non-empty only for the Hive proxy\n (and only when ``api_key`` is provided).\n \"\"\"\n extra_headers: dict[str, str] = {}\n if model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n # Hive proxy expects Bearer auth; litellm's Anthropic handler\n # only sends x-api-key without this nudge.\n if api_key:\n extra_headers[\"Authorization\"] = f\"Bearer {api_key}\"\n return model, api_base, extra_headers\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n\n# Directory for dumping failed requests. Resolved lazily so HIVE_HOME\n# overrides (set by the desktop shell) take effect even if this module\n# is imported before framework.config picks up the override.\ndef _failed_requests_dir() -> Path:\n from framework.config import HIVE_HOME\n\n return HIVE_HOME / \"failed_requests\"\n\n\n# Maximum number of dump files to retain in $HIVE_HOME/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(model, input_tokens, output_tokens, cache_read, cache_creation)\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens)\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (getattr(_details, \"cache_write_tokens\", 0) or 0 if _details is not None else 0) or (\n getattr(usage, \"cache_creation_input_tokens\", 0) or 0\n )\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n _failed_requests_dir().glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n dump_dir = _failed_requests_dir()\n dump_dir.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = dump_dir / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {_failed_requests_dir()}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n self._hive_proxy_auth = bool(_original_model.lower().startswith(\"hive/\"))\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n self._hive_proxy_auth = bool(_original_model.lower().startswith(\"hive/\"))\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # The Hive LLM proxy fronts GLM, which drifts into \"explain\n # the plan\" mode on long-context turns instead of emitting\n # tool_use blocks (verified 2026-04-28: tool_choice=null →\n # text-only stop=stop; tool_choice=required → clean\n # tool_use). Force a tool call when tools are available\n # so queens can't get stuck in chat mode. Callers that\n # legitimately want a non-tool turn can override via\n # extra_kwargs.\n kwargs.setdefault(\"tool_choice\", \"required\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # See `complete()` for the rationale: GLM behind the Hive\n # proxy needs forcing or it goes chat-mode on long contexts.\n kwargs.setdefault(\"tool_choice\", \"required\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n\n from framework.config import HIVE_HOME as _HIVE_HOME\n\n _debug_dir = _HIVE_HOME / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # See `complete()` for the rationale: GLM behind the Hive\n # proxy needs forcing or it goes chat-mode on long contexts.\n kwargs.setdefault(\"tool_choice\", \"required\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "73511a3c59de873ad91746e5d21f561b73e0b949": {"t": 1785765515.1395378, "n": 123773, "h": "029a7fdd3da62c61", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _api_base_needs_bearer_auth(api_base: str | None) -> bool:\n \"\"\"Return True when api_base points at an Anthropic-compatible endpoint\n that authenticates via ``Authorization: Bearer`` rather than ``x-api-key``.\n\n The Hive LLM proxy (Rust service in hive-backend/llm/) speaks the\n Anthropic Messages API but mints user-scoped JWTs and validates them\n via Bearer auth. Default upstream Anthropic endpoints (api.anthropic.com,\n Kimi's api.kimi.com/coding) keep using x-api-key, so the override is\n scoped to known hive-proxy hosts plus the env-configured override.\n \"\"\"\n if not api_base:\n return False\n # Strip protocol, port, and path so a plain hostname compare is enough\n # for the common cases.\n lowered = api_base.lower()\n for host in (\"adenhq.com\", \"open-hive.com\", \"127.0.0.1:8890\", \"localhost:8890\"):\n if host in lowered:\n return True\n override = os.environ.get(\"HIVE_LLM_BASE_URL\")\n if override and override.lower() in lowered:\n return True\n return False\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\ndef rewrite_proxy_model(\n model: str, api_key: str | None, api_base: str | None\n) -> tuple[str, str | None, dict[str, str]]:\n \"\"\"Apply Hive/Kimi proxy rewrites for any caller of ``litellm.acompletion``.\n\n Both the Hive LLM proxy and Kimi For Coding expose Anthropic-API-\n compatible endpoints. LiteLLM doesn't recognise the ``hive/`` or\n ``kimi/`` prefixes natively, so we rewrite them to ``anthropic/``\n here. For the Hive proxy we also stamp a Bearer token into\n ``extra_headers`` because litellm's Anthropic handler only sends\n ``x-api-key`` and the proxy expects ``Authorization: Bearer``.\n\n Used by ad-hoc ``litellm.acompletion`` callers (e.g. the vision-\n fallback subagent in ``caption_tool_image``) so they hit the same\n proxy with the same auth as the main agent's ``LiteLLMProvider``.\n The provider's own ``__init__`` keeps its inlined rewrite for now —\n this helper is the single source of truth for ad-hoc callers.\n\n Returns: (rewritten_model, normalised_api_base, extra_headers).\n The ``extra_headers`` dict is non-empty only for the Hive proxy\n (and only when ``api_key`` is provided).\n \"\"\"\n extra_headers: dict[str, str] = {}\n if model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n # Hive proxy expects Bearer auth; litellm's Anthropic handler\n # only sends x-api-key without this nudge.\n if api_key:\n extra_headers[\"Authorization\"] = f\"Bearer {api_key}\"\n return model, api_base, extra_headers\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n\n# Directory for dumping failed requests. Resolved lazily so HIVE_HOME\n# overrides (set by the desktop shell) take effect even if this module\n# is imported before framework.config picks up the override.\ndef _failed_requests_dir() -> Path:\n from framework.config import HIVE_HOME\n\n return HIVE_HOME / \"failed_requests\"\n\n\n# Maximum number of dump files to retain in $HIVE_HOME/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(model, input_tokens, output_tokens, cache_read, cache_creation)\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens)\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (getattr(_details, \"cache_write_tokens\", 0) or 0 if _details is not None else 0) or (\n getattr(usage, \"cache_creation_input_tokens\", 0) or 0\n )\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n _failed_requests_dir().glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n dump_dir = _failed_requests_dir()\n dump_dir.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = dump_dir / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {_failed_requests_dir()}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n self._hive_proxy_auth = bool(_original_model.lower().startswith(\"hive/\"))\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n self._hive_proxy_auth = bool(_original_model.lower().startswith(\"hive/\"))\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # The Hive LLM proxy fronts GLM, which drifts into \"explain\n # the plan\" mode on long-context turns instead of emitting\n # tool_use blocks (verified 2026-04-28: tool_choice=null →\n # text-only stop=stop; tool_choice=required → clean\n # tool_use). Force a tool call when tools are available\n # so queens can't get stuck in chat mode. Callers that\n # legitimately want a non-tool turn can override via\n # extra_kwargs.\n kwargs.setdefault(\"tool_choice\", \"required\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # See `complete()` for the rationale: GLM behind the Hive\n # proxy needs forcing or it goes chat-mode on long contexts.\n kwargs.setdefault(\"tool_choice\", \"required\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n\n from framework.config import HIVE_HOME as _HIVE_HOME\n\n _debug_dir = _HIVE_HOME / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n elif self._hive_proxy_auth:\n # See `complete()` for the rationale: GLM behind the Hive\n # proxy needs forcing or it goes chat-mode on long contexts.\n kwargs.setdefault(\"tool_choice\", \"required\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "7b0b4721677c140a929873cdc00704b4da46fae3": {"t": 1785765516.892419, "n": 120607, "h": "5ccedac309b8d1e6", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _api_base_needs_bearer_auth(api_base: str | None) -> bool:\n \"\"\"Return True when api_base points at an Anthropic-compatible endpoint\n that authenticates via ``Authorization: Bearer`` rather than ``x-api-key``.\n\n The Hive LLM proxy (Rust service in hive-backend/llm/) speaks the\n Anthropic Messages API but mints user-scoped JWTs and validates them\n via Bearer auth. Default upstream Anthropic endpoints (api.anthropic.com,\n Kimi's api.kimi.com/coding) keep using x-api-key, so the override is\n scoped to known hive-proxy hosts plus the env-configured override.\n \"\"\"\n if not api_base:\n return False\n # Strip protocol, port, and path so a plain hostname compare is enough\n # for the common cases.\n lowered = api_base.lower()\n for host in (\"adenhq.com\", \"open-hive.com\", \"127.0.0.1:8890\", \"localhost:8890\"):\n if host in lowered:\n return True\n override = os.environ.get(\"HIVE_LLM_BASE_URL\")\n if override and override.lower() in lowered:\n return True\n return False\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n\n# Directory for dumping failed requests. Resolved lazily so HIVE_HOME\n# overrides (set by the desktop shell) take effect even if this module\n# is imported before framework.config picks up the override.\ndef _failed_requests_dir() -> Path:\n from framework.config import HIVE_HOME\n\n return HIVE_HOME / \"failed_requests\"\n\n\n# Maximum number of dump files to retain in $HIVE_HOME/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(model, input_tokens, output_tokens, cache_read, cache_creation)\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens)\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (getattr(_details, \"cache_write_tokens\", 0) or 0 if _details is not None else 0) or (\n getattr(usage, \"cache_creation_input_tokens\", 0) or 0\n )\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n _failed_requests_dir().glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n dump_dir = _failed_requests_dir()\n dump_dir.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = dump_dir / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {_failed_requests_dir()}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n\n from framework.config import HIVE_HOME as _HIVE_HOME\n\n _debug_dir = _HIVE_HOME / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "ae2aa30edfe185f6790f8fd18e93c00dedf99460": {"t": 1785765519.541186, "n": 120606, "h": "eeed1516551de3c1", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _api_base_needs_bearer_auth(api_base: str | None) -> bool:\n \"\"\"Return True when api_base points at an Anthropic-compatible endpoint\n that authenticates via ``Authorization: Bearer`` rather than ``x-api-key``.\n\n The Hive LLM proxy (Rust service in hive-backend/llm/) speaks the\n Anthropic Messages API but mints user-scoped JWTs and validates them\n via Bearer auth. Default upstream Anthropic endpoints (api.anthropic.com,\n Kimi's api.kimi.com/coding) keep using x-api-key, so the override is\n scoped to known hive-proxy hosts plus the env-configured override.\n \"\"\"\n if not api_base:\n return False\n # Strip protocol, port, and path so a plain hostname compare is enough\n # for the common cases.\n lowered = api_base.lower()\n for host in (\"adenhq.com\", \"open-hive.com\", \"127.0.0.1:8890\", \"localhost:8890\"):\n if host in lowered:\n return True\n override = os.environ.get(\"HIVE_LLM_BASE_URL\")\n if override and override.lower() in lowered:\n return True\n return False\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests. Resolved lazily so HIVE_HOME\n# overrides (set by the desktop shell) take effect even if this module\n# is imported before framework.config picks up the override.\ndef _failed_requests_dir() -> Path:\n from framework.config import HIVE_HOME\n\n return HIVE_HOME / \"failed_requests\"\n\n\n# Maximum number of dump files to retain in $HIVE_HOME/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(model, input_tokens, output_tokens, cache_read, cache_creation)\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens)\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (getattr(_details, \"cache_write_tokens\", 0) or 0 if _details is not None else 0) or (\n getattr(usage, \"cache_creation_input_tokens\", 0) or 0\n )\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n _failed_requests_dir().glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n dump_dir = _failed_requests_dir()\n dump_dir.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = dump_dir / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {_failed_requests_dir()}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n\n from framework.config import HIVE_HOME as _HIVE_HOME\n\n _debug_dir = _HIVE_HOME / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "7c3bada70cf588b3039b0c54da27d93615c3dca7": {"t": 1785765521.6060615, "n": 121028, "h": "f609a4bfd28b9275", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _api_base_needs_bearer_auth(api_base: str | None) -> bool:\n \"\"\"Return True when ``api_base`` points at an Anthropic-compatible endpoint\n that authenticates via ``Authorization: Bearer`` rather than ``x-api-key``.\n\n The hive-llm proxy (Rust service in hive-backend/llm/) speaks the Anthropic\n Messages API but mints user-scoped JWTs and validates them via Bearer auth.\n Default upstream Anthropic endpoints (api.anthropic.com, Kimi's\n api.kimi.com/coding) keep using x-api-key, so the override is scoped to the\n known hive-proxy hosts plus the env-configured override.\n \"\"\"\n if not api_base:\n return False\n lowered = api_base.lower()\n for host in (\"adenhq.com\", \"open-hive.com\", \"127.0.0.1:8890\", \"localhost:8890\"):\n if host in lowered:\n return True\n override = os.environ.get(\"HIVE_LLM_BASE_URL\")\n if override and override.lower() in lowered:\n return True\n return False\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth/JWT token handling.\n\n Two cases are remapped:\n 1. **Anthropic OAuth tokens** (``sk-ant-oat`` prefix). litellm puts the token\n into ``x-api-key`` but Anthropic's API requires it on\n ``Authorization: Bearer`` only — see BerriAI/litellm#19618.\n 2. **Hive LLM proxy bearer tokens** (any JWT). The Rust proxy at\n hive-backend/llm/ speaks the Anthropic Messages API but authenticates\n with ``Authorization: Bearer <jwt>``; litellm's default ``x-api-key``\n would 401.\n\n Both cases share the same fix: promote whatever's in ``x-api-key`` to\n ``Authorization: Bearer`` and drop ``x-api-key``.\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n # The hive-llm proxy speaks the Anthropic Messages API but authenticates\n # via Authorization: Bearer <jwt>; x-api-key is ignored and returns\n # missing_auth. Promote x-api-key → Authorization for those endpoints\n # so hive's per-user stream JWT actually authenticates the request.\n hive_needs_remap = (\n x_api_key\n and not auth\n and _api_base_needs_bearer_auth(api_base)\n )\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n elif hive_needs_remap:\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {x_api_key}\"\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(model, input_tokens, output_tokens, cache_read, cache_creation)\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens)\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (getattr(_details, \"cache_write_tokens\", 0) or 0 if _details is not None else 0) or (\n getattr(usage, \"cache_creation_input_tokens\", 0) or 0\n )\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "22d75bfb05b92855ea5cc6c3914c82f346a000e4": {"t": 1785765523.4757316, "n": 119304, "h": "c39f7a75493b3fcb", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(model, input_tokens, output_tokens, cache_read, cache_creation)\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens)\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (getattr(_details, \"cache_write_tokens\", 0) or 0 if _details is not None else 0) or (\n getattr(usage, \"cache_creation_input_tokens\", 0) or 0\n )\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "386bbd57804719e1124ac30e47ebbc6fd329bc6c": {"t": 1785765525.1352725, "n": 119422, "h": "850942815bd8971e", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.model_catalog import get_model_pricing\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _cost_from_catalog_pricing(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Last-resort cost calculation using curated catalog pricing.\n\n Consulted only when the provider response carries no native cost and\n LiteLLM's own catalog has no pricing for ``model``. Reads\n ``pricing_usd_per_mtok`` from ``model_catalog.json``. Rates are USD per\n million tokens.\n\n ``cached_tokens`` and ``cache_creation_tokens`` are subsets of\n ``input_tokens`` (see ``_extract_cache_tokens``), so subtract them from\n the base input count to avoid double-billing. If a cache rate is absent,\n fall back to the plain input rate.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n pricing = get_model_pricing(model)\n if pricing is None and \"/\" in model:\n # LiteLLM prefixes some ids (e.g. \"openrouter/z-ai/glm-5.1\"); the\n # catalog stores the bare form (\"z-ai/glm-5.1\"). Strip one segment.\n pricing = get_model_pricing(model.split(\"/\", 1)[1])\n if pricing is None:\n return 0.0\n\n per_mtok_in = pricing.get(\"input\", 0.0)\n per_mtok_out = pricing.get(\"output\", 0.0)\n per_mtok_cache_read = pricing.get(\"cache_read\", per_mtok_in)\n per_mtok_cache_write = pricing.get(\"cache_creation\", per_mtok_in)\n\n plain_input = max(input_tokens - cached_tokens - cache_creation_tokens, 0)\n total = (\n plain_input * per_mtok_in\n + cached_tokens * per_mtok_cache_read\n + cache_creation_tokens * per_mtok_cache_write\n + output_tokens * per_mtok_out\n ) / 1_000_000\n return float(total) if total > 0 else 0.0\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n 4. ``pricing_usd_per_mtok`` from the curated model catalog — covers\n models (e.g. GLM, Kimi, MiniMax) that LiteLLM doesn't price.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n\n if usage is not None:\n input_tokens = int(getattr(usage, \"prompt_tokens\", 0) or 0)\n output_tokens = int(getattr(usage, \"completion_tokens\", 0) or 0)\n cache_read, cache_creation = _extract_cache_tokens(usage)\n fallback = _cost_from_catalog_pricing(\n model, input_tokens, output_tokens, cache_read, cache_creation\n )\n if fallback > 0:\n return fallback\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n if total > 0:\n return float(total)\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return _cost_from_catalog_pricing(\n model, input_tokens, output_tokens, cached_tokens, cache_creation_tokens\n )\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (\n getattr(_details, \"cache_write_tokens\", 0) or 0\n if _details is not None\n else 0\n ) or (getattr(usage, \"cache_creation_input_tokens\", 0) or 0)\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d \"\n \"cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "77cc1696068cc7107f9d5ad05901380a4feae23e": {"t": 1785765527.9298387, "n": 116976, "h": "d81961e29a71e368", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _extract_cost(response: Any, model: str) -> float:\n \"\"\"Pull the USD cost for a non-streaming completion response.\n\n Sources checked, in priority order:\n 1. ``usage.cost`` — populated when OpenRouter returns native cost via\n ``usage: {include: true}`` or when ``litellm.include_cost_in_streaming_usage``\n is on.\n 2. ``response._hidden_params[\"response_cost\"]`` — set by LiteLLM's\n logging layer after most successful completions.\n 3. ``litellm.completion_cost(...)`` — computes from the model pricing\n table; works across Anthropic, OpenAI, and OpenRouter as long as the\n model is in LiteLLM's catalog.\n\n Returns 0.0 for unpriced models or unexpected response shapes — cost is a\n display concern, never let it break the hot path. For streaming paths\n where the aggregate response isn't a full ``ModelResponse``, use\n :func:`_cost_from_tokens` with the already-extracted token counts.\n \"\"\"\n if response is None:\n return 0.0\n usage = getattr(response, \"usage\", None)\n usage_cost = getattr(usage, \"cost\", None) if usage is not None else None\n if isinstance(usage_cost, (int, float)) and usage_cost > 0:\n return float(usage_cost)\n\n hidden = getattr(response, \"_hidden_params\", None)\n if isinstance(hidden, dict):\n hp_cost = hidden.get(\"response_cost\")\n if isinstance(hp_cost, (int, float)) and hp_cost > 0:\n return float(hp_cost)\n\n try:\n import litellm as _litellm\n\n computed = _litellm.completion_cost(completion_response=response, model=model)\n if isinstance(computed, (int, float)) and computed > 0:\n return float(computed)\n except Exception as exc:\n logger.debug(\"[cost] completion_cost failed for %s: %s\", model, exc)\n return 0.0\n\n\ndef _cost_from_tokens(\n model: str,\n input_tokens: int,\n output_tokens: int,\n cached_tokens: int = 0,\n cache_creation_tokens: int = 0,\n) -> float:\n \"\"\"Compute USD cost from already-normalized token counts.\n\n Used on streaming paths where the aggregate ``response`` is the stream\n wrapper (not a full ``ModelResponse``) and ``litellm.completion_cost`` on\n it either no-ops or raises. Calls ``litellm.cost_per_token`` directly\n with the cache-aware inputs so Anthropic's 5-min-write / cache-read\n multipliers are applied correctly.\n \"\"\"\n if not model or (input_tokens == 0 and output_tokens == 0):\n return 0.0\n try:\n import litellm as _litellm\n\n prompt_cost, completion_cost = _litellm.cost_per_token(\n model=model,\n prompt_tokens=input_tokens,\n completion_tokens=output_tokens,\n cache_read_input_tokens=cached_tokens,\n cache_creation_input_tokens=cache_creation_tokens,\n )\n total = (prompt_cost or 0.0) + (completion_cost or 0.0)\n return float(total) if total > 0 else 0.0\n except Exception as exc:\n logger.debug(\"[cost] cost_per_token failed for %s: %s\", model, exc)\n return 0.0\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (\n getattr(_details, \"cache_write_tokens\", 0) or 0\n if _details is not None\n else 0\n ) or (getattr(usage, \"cache_creation_input_tokens\", 0) or 0)\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n cost_usd = _extract_cost(response, self.model)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n cost_usd=response.cost_usd,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d \"\n \"cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n cost_usd = _cost_from_tokens(\n self.model,\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n cost_usd=cost_usd,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "8c6428f445cf43f7d2824e5116b7d81c77f35ee8": {"t": 1785765529.5544589, "n": 112848, "h": "d5417238f2593963", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n# OpenRouter sub-provider prefixes whose upstream API honors `cache_control`.\n# OpenRouter passes the marker through to the underlying provider for these.\n# (See https://openrouter.ai/docs/guides/best-practices/prompt-caching.)\n# OpenAI/DeepSeek/Groq/Grok/Moonshot route through OpenRouter but cache\n# automatically server-side — sending cache_control there is a no-op, not a\n# win, and they need a separate prefix-stability fix to actually get hits.\n_OPENROUTER_CACHE_CONTROL_PREFIXES = (\n \"openrouter/anthropic/\",\n \"openrouter/google/gemini-\",\n \"openrouter/z-ai/glm\",\n \"openrouter/minimax/\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n if any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES):\n return True\n return any(model.startswith(p) for p in _OPENROUTER_CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _extract_cache_tokens(usage: Any) -> tuple[int, int]:\n \"\"\"Pull (cache_read, cache_creation) from a LiteLLM usage object.\n\n Both are subsets of ``prompt_tokens`` already — providers count them\n inside the input total. Surface separately for visibility, never sum.\n\n Field names vary by provider/proxy; check the known shapes in priority\n order and fall back to 0:\n\n cache_read:\n - ``prompt_tokens_details.cached_tokens`` — OpenAI-shape; also what\n LiteLLM normalizes Anthropic and OpenRouter into.\n - ``cache_read_input_tokens`` — raw Anthropic field name.\n\n cache_creation:\n - ``prompt_tokens_details.cache_write_tokens`` — OpenRouter's\n normalized field for cache writes (verified empirically against\n ``openrouter/anthropic/*`` and ``openrouter/z-ai/*`` responses).\n - ``cache_creation_input_tokens`` — raw Anthropic top-level field.\n \"\"\"\n if not usage:\n return 0, 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cache_read = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n cache_creation = (\n getattr(_details, \"cache_write_tokens\", 0) or 0\n if _details is not None\n else 0\n ) or (getattr(usage, \"cache_creation_input_tokens\", 0) or 0)\n return cache_read, cache_creation\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n system_dynamic_suffix: str | None = None,\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n # If the routed sub-provider honors cache_control (e.g.\n # openrouter/anthropic/*), split the static prefix from the dynamic\n # suffix so the prefix stays cache-warm across turns. Otherwise fall\n # back to a single concatenated string.\n system_message = _build_system_message(\n compat_system,\n system_dynamic_suffix,\n self.model,\n )\n\n full_messages: list[dict[str, Any]] = []\n if system_message is not None:\n full_messages.append(system_message)\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n When the routed sub-provider honors ``cache_control`` (e.g.\n ``openrouter/anthropic/*``), the message builder splits the static\n prefix from the dynamic suffix so the prefix stays cache-warm.\n Otherwise the suffix is concatenated into a single system string.\n \"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(\n messages,\n system,\n tools,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n cached_tokens=response.cached_tokens,\n cache_creation_tokens=response.cache_creation_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n cache_creation_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n cached_tokens, cache_creation_tokens = _extract_cache_tokens(usage)\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d \"\n \"cache_creation=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n # `calculate_total_usage` aggregates token totals\n # but discards `prompt_tokens_details` — which is\n # where OpenRouter puts `cached_tokens` and\n # `cache_write_tokens`. Recover them directly\n # from the most recent chunk that carries usage.\n cached_tokens, cache_creation_tokens = 0, 0\n for _raw in reversed(_chunks):\n _raw_usage = getattr(_raw, \"usage\", None)\n if _raw_usage is None:\n continue\n _cr, _cc = _extract_cache_tokens(_raw_usage)\n if _cr or _cc:\n cached_tokens, cache_creation_tokens = _cr, _cc\n break\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d \"\n \"cached=%d cache_creation=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n cache_creation_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n cached_tokens = 0\n cache_creation_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n cached_tokens = event.cached_tokens\n cache_creation_tokens = event.cache_creation_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n cache_creation_tokens=cache_creation_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "a70f92edbe75a0c4f9d847f34a9738b279311e07": {"t": 1785765531.2298896, "n": 108416, "h": "9c8d281b3bf0d483", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n OpenRouter models don't honor ``cache_control`` on content blocks, so\n when a ``system_dynamic_suffix`` is provided we simply concatenate it\n onto ``system`` before building the compat messages — behaviorally\n identical to today's single-string path.\n \"\"\"\n if system_dynamic_suffix:\n system = f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "390038225b0a25f9e3e9839fe222e5b04ad63d57": {"t": 1785765533.476127, "n": 108472, "h": "45553b917995e7d9", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\ndef _build_system_message(\n system: str,\n system_dynamic_suffix: str | None,\n model: str,\n) -> dict[str, Any] | None:\n \"\"\"Construct the system-role message for the chat completion.\n\n Returns ``None`` when there is nothing to send.\n\n Two-block split path — used when the caller supplied a non-empty\n ``system_dynamic_suffix`` AND the provider honors ``cache_control``\n (Anthropic, MiniMax, Z-AI/GLM). We emit ``content`` as a list of two\n text blocks with an ephemeral ``cache_control`` marker on the first\n block only. The prompt cache keeps the static prefix warm across\n turns and across iterations within a turn; only the small dynamic\n tail is recomputed on every request.\n\n Single-string path — used for every other case (no suffix provided,\n or provider doesn't honor ``cache_control``). We concatenate\n ``system`` + ``\\\\n\\\\n`` + ``system_dynamic_suffix`` and attach\n ``cache_control`` to the whole message when the provider supports\n it. This is byte-identical to the pre-split behavior for all\n non-cache-control providers (OpenAI, Gemini, Groq, Ollama, etc.).\n \"\"\"\n if not system and not system_dynamic_suffix:\n return None\n if system_dynamic_suffix and _model_supports_cache_control(model):\n content_blocks: list[dict[str, Any]] = []\n if system:\n content_blocks.append(\n {\n \"type\": \"text\",\n \"text\": system,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n )\n content_blocks.append({\"type\": \"text\", \"text\": system_dynamic_suffix})\n return {\"role\": \"system\", \"content\": content_blocks}\n # Single-string path (legacy or no-cache-control provider).\n combined = system\n if system_dynamic_suffix:\n combined = (\n f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n )\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": combined}\n if _model_supports_cache_control(model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n return sys_msg\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. When set and\n the provider honors ``cache_control``, ``system`` is sent as the\n cached prefix and the suffix trails as an uncached second content\n block. Otherwise the two strings are concatenated into a single\n system message (legacy behavior).\n \"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\n\n OpenRouter models don't honor ``cache_control`` on content blocks, so\n when a ``system_dynamic_suffix`` is provided we simply concatenate it\n onto ``system`` before building the compat messages — behaviorally\n identical to today's single-string path.\n \"\"\"\n if system_dynamic_suffix:\n system = (\n f\"{system}\\n\\n{system_dynamic_suffix}\" if system else system_dynamic_suffix\n )\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n system_dynamic_suffix: str | None = None,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n\n ``system_dynamic_suffix`` is an optional per-turn tail. See\n ``acomplete`` docstring for the two-block split semantics.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n system_dynamic_suffix=system_dynamic_suffix,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n sys_msg = _build_system_message(system, system_dynamic_suffix, self.model)\n if sys_msg is not None:\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "a881fe68da40e5706d0cc9d84e9d820638eacbdf": {"t": 1785765535.0960176, "n": 105061, "h": "0675e974589bf7b8", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n # Pass store directly to OpenAI in case litellm drops it as unknown\n if \"extra_body\" not in kwargs:\n kwargs[\"extra_body\"] = {}\n kwargs[\"extra_body\"][\"store\"] = False\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "589c5b06fe2ecf529c1a9e4dd9cd1e636b640157": {"t": 1785765536.7374794, "n": 104846, "h": "7e45d90791f96564", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\"LiteLLM is not installed. Please install it with: uv pip install litellm\")\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base)\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(self, max_retries: int | None = None, **kwargs: Any) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\"[async-retry] Empty response after assistant message — expected, not retrying.\")\n return response\n\n finish_reason = response.choices[0].finish_reason if response.choices else \"unknown\"\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\")\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (message.get(\"role\") == \"assistant\" and not message.get(\"content\") and not message.get(\"tool_calls\"))\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from datetime import datetime as _dt\n from pathlib import Path as _Path\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(_json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\")\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\"))\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback: input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "6979ea825d7dda33aa74f44a8d24d8e91226dfe6": {"t": 1785765538.3517385, "n": 105531, "h": "bfc2cad9fbddaeb1", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\n# Separate, much lower cap for \"empty response, finish_reason=stop\"\n# scenarios. Unlike a real 429, these are rarely transient: Gemini\n# returns stop+empty on silently-filtered safety blocks, poisoned\n# conversation state (dangling tool_result after compaction), or\n# malformed tool schemas. Waiting minutes doesn't fix any of those, so\n# give up after 3 attempts (2+4+8 = 14s) and surface an actionable\n# error instead of burning 12+ minutes on exponential backoff.\nEMPTY_RESPONSE_MAX_RETRIES = 3\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(\n self, model: str, api_key: str | None = None, api_base: str | None = None\n ) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n # Use a much lower retry cap for empty-response\n # recoveries than for real exceptions. These are\n # almost never transient (see EMPTY_RESPONSE_MAX_RETRIES\n # rationale at the top of the file).\n empty_cap = min(retries, EMPTY_RESPONSE_MAX_RETRIES)\n if attempt >= empty_cap:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after \"\n f\"{attempt + 1} attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"This is almost never a rate limit despite the \"\n f\"earlier log message — check the dumped request \"\n f\"at {dump_path} for poisoned conversation state \"\n f\"(dangling tool_result after compaction), a \"\n f\"safety-filter trigger in the prompt, or a \"\n f\"malformed tool schema.\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}). \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{empty_cap}). \"\n f\"Note: empty-response retries are capped at \"\n f\"{EMPTY_RESPONSE_MAX_RETRIES} because this is rarely \"\n f\"a transient rate limit on small payloads.\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from pathlib import Path as _Path\n from datetime import datetime as _dt\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(\n _json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\"\n )\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "0964758b12c9f980316e60b7c9bda08aca23be40": {"t": 1785765539.950884, "n": 103418, "h": "e31993452b0cda01", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(\n self, model: str, api_key: str | None = None, api_base: str | None = None\n ) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from pathlib import Path as _Path\n from datetime import datetime as _dt\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(\n _json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\"\n )\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "9ad95fde5988243a09c89911bb8c2bf10d46ea23": {"t": 1785765541.5550208, "n": 101754, "h": "f314c7922f0d1fdd", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(\n self, model: str, api_key: str | None = None, api_base: str | None = None\n ) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "4be61ebfc76ea262035cb279b379299ed3a9f67a": {"t": 1785765543.399144, "n": 96855, "h": "150e3f95804707ae", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\nlogging.getLogger(\"openai._base_client\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpx\").setLevel(logging.WARNING)\nlogging.getLogger(\"httpcore\").setLevel(logging.WARNING)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(\n self, model: str, api_key: str | None = None, api_base: str | None = None\n ) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if logger.isEnabledFor(logging.DEBUG) and full_messages:\n import json as _json\n from pathlib import Path as _Path\n from datetime import datetime as _dt\n\n _debug_dir = _Path.home() / \".hive\" / \"debug_logs\"\n _debug_dir.mkdir(parents=True, exist_ok=True)\n _ts = _dt.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n _dump_file = _debug_dir / f\"llm_request_{_ts}.json\"\n _summary = []\n for _mi, _m in enumerate(full_messages):\n _role = _m.get(\"role\", \"?\")\n _c = _m.get(\"content\")\n _tc = _m.get(\"tool_calls\")\n _tcid = _m.get(\"tool_call_id\")\n _summary.append(\n {\n \"idx\": _mi,\n \"role\": _role,\n \"content_length\": len(str(_c)) if _c else 0,\n \"content_preview\": str(_c)[:200] if _c else repr(_c),\n \"has_tool_calls\": bool(_tc),\n \"tool_call_count\": len(_tc) if _tc else 0,\n \"tool_call_id\": _tcid,\n }\n )\n try:\n _dump_file.write_text(\n _json.dumps(_summary, indent=2, ensure_ascii=False), encoding=\"utf-8\"\n )\n logger.debug(\"[LLM-MSG] %d messages dumped to %s\", len(full_messages), _dump_file)\n except Exception:\n pass\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "7a2752eb4286de5877546382a06ec32342569ac8": {"t": 1785765544.912028, "n": 101738, "h": "06ed3fcd934d1008", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"api_base\": kwargs.get(\"api_base\"),\n \"request_keys\": sorted(kwargs.keys()),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n \"stream\": kwargs.get(\"stream\"),\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": kwargs.get(\"response_format\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _summarize_message_content(content: Any) -> dict[str, Any]:\n \"\"\"Return a structural summary of one message content payload.\"\"\"\n if isinstance(content, str):\n return {\n \"content_kind\": \"string\",\n \"text_chars\": len(content),\n }\n\n if isinstance(content, list):\n block_types: list[str] = []\n text_chars = 0\n for block in content:\n if isinstance(block, dict):\n block_type = str(block.get(\"type\", \"unknown\"))\n block_types.append(block_type)\n if block_type == \"text\":\n text_chars += len(str(block.get(\"text\", \"\")))\n elif block_type == \"tool_result\":\n block_content = block.get(\"content\")\n if isinstance(block_content, str):\n text_chars += len(block_content)\n elif isinstance(block_content, list):\n for inner in block_content:\n if isinstance(inner, dict) and inner.get(\"type\") == \"text\":\n text_chars += len(str(inner.get(\"text\", \"\")))\n else:\n block_types.append(type(block).__name__)\n return {\n \"content_kind\": \"list\",\n \"blocks\": len(content),\n \"block_types\": block_types,\n \"text_chars\": text_chars,\n }\n\n return {\n \"content_kind\": type(content).__name__,\n }\n\n\ndef _summarize_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:\n \"\"\"Build a high-signal, no-secret summary of the outgoing messages payload.\"\"\"\n summary: list[dict[str, Any]] = []\n for idx, message in enumerate(messages):\n item: dict[str, Any] = {\n \"idx\": idx,\n \"role\": message.get(\"role\"),\n \"keys\": sorted(message.keys()),\n }\n item.update(_summarize_message_content(message.get(\"content\")))\n tool_calls = message.get(\"tool_calls\")\n if isinstance(tool_calls, list):\n item[\"tool_calls\"] = len(tool_calls)\n tool_names = []\n for tc in tool_calls:\n if isinstance(tc, dict):\n fn = tc.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n if tool_names:\n item[\"tool_call_names\"] = tool_names\n if message.get(\"cache_control\"):\n item[\"cache_control\"] = True\n if message.get(\"tool_call_id\"):\n item[\"tool_call_id\"] = str(message.get(\"tool_call_id\"))\n summary.append(item)\n return summary\n\n\ndef _summarize_request_for_log(kwargs: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Return a compact structural summary of a LiteLLM request payload.\"\"\"\n tools = kwargs.get(\"tools\")\n tool_names: list[str] = []\n if isinstance(tools, list):\n for tool in tools:\n if isinstance(tool, dict):\n fn = tool.get(\"function\")\n if isinstance(fn, dict) and fn.get(\"name\"):\n tool_names.append(str(fn[\"name\"]))\n\n messages = kwargs.get(\"messages\", [])\n if isinstance(messages, list):\n non_system_roles = [m.get(\"role\") for m in messages if m.get(\"role\") != \"system\"]\n else:\n non_system_roles = []\n return {\n \"model\": kwargs.get(\"model\"),\n \"api_base\": kwargs.get(\"api_base\"),\n \"stream\": kwargs.get(\"stream\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"tool_count\": len(tools) if isinstance(tools, list) else 0,\n \"tool_names\": tool_names,\n \"tool_choice\": kwargs.get(\"tool_choice\"),\n \"response_format\": bool(kwargs.get(\"response_format\")),\n \"message_count\": len(messages) if isinstance(messages, list) else 0,\n \"non_system_message_count\": len(non_system_roles),\n \"first_non_system_role\": non_system_roles[0] if non_system_roles else None,\n \"last_non_system_role\": non_system_roles[-1] if non_system_roles else None,\n \"system_only\": bool(messages) and not non_system_roles,\n \"messages\": _summarize_messages_for_log(messages if isinstance(messages, list) else []),\n }\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _is_zai_openai_backend(self) -> bool:\n \"\"\"Return True when using Z-AI's OpenAI-compatible chat endpoint.\"\"\"\n model = (self.model or \"\").lower()\n api_base = (self.api_base or \"\").lower()\n return \"api.z.ai\" in api_base or model.startswith(\"openai/glm-\") or model == \"glm-5\"\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n request_summary = _summarize_request_for_log(kwargs)\n logger.debug(\n \"[stream] prepared request: %s\",\n json.dumps(request_summary, default=str),\n )\n if request_summary[\"system_only\"]:\n logger.warning(\n \"[stream] %s request has no non-system chat messages \"\n \"(api_base=%s tools=%d system_chars=%d). \"\n \"Some chat-completions backends reject system-only payloads.\",\n self.model,\n self.api_base,\n request_summary[\"tool_count\"],\n sum(\n message.get(\"text_chars\", 0)\n for message in request_summary[\"messages\"]\n if message.get(\"role\") == \"system\"\n ),\n )\n if self._is_zai_openai_backend():\n logger.warning(\n \"[stream] %s appears to be using Z-AI/GLM's OpenAI-compatible backend. \"\n \"This backend has rejected system-only payloads with \"\n \"'The messages parameter is illegal.' in prior requests.\",\n self.model,\n )\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=f\"stream_exception_{type(e).__name__.lower()}\",\n attempt=attempt,\n )\n logger.error(\n \"[stream] %s request failed with %s: %s | request=%s | dump=%s\",\n self.model,\n type(e).__name__,\n e,\n json.dumps(_summarize_request_for_log(kwargs), default=str),\n dump_path,\n )\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "7e1ebf1c26c59303959b1060fc3729fa7858415c": {"t": 1785765546.7447143, "n": 95175, "h": "a0429994c8cf9b5a", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "ecbf543e4c54b47f8008d7a3ae782dc5f7e517ff": {"t": 1785765548.374518, "n": 92163, "h": "f96bd4ae3241ebb9", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "d8712ceb7215dec92f4eadb9da313405e2ad186d": {"t": 1785765549.9806607, "n": 90547, "h": "e4e627f5dc9ae1b4", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n try:\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n except OSError as e:\n logger.warning(f\"Failed to dump request debug log to {FAILED_REQUESTS_DIR}: {e}\")\n return \"log_write_failed\"\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "6b86c602c7917655c24804c199b7c922a74e110a": {"t": 1785765551.5440679, "n": 93309, "h": "a979ebf86d55a130", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "e7506fcd25737e79e38eed079e04d65643c660c0": {"t": 1785765554.0033605, "n": 91913, "h": "4a4ac5a46ad0acaf", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n def reconfigure(self, model: str, api_key: str | None = None, api_base: str | None = None) -> None:\n \"\"\"Hot-swap the model, API key, and/or base URL on this provider instance.\n\n Since the same LiteLLMProvider object is shared by reference across the\n session, queen runner, agent runtime, and execution streams, mutating\n these attributes in-place propagates to all callers on the next LLM call.\n \"\"\"\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\"):]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "d10912be15594ef5f4e0746c0b4da8312bcabf6d": {"t": 1785765555.7446163, "n": 93053, "h": "a9b359635f43b192", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nfrom __future__ import annotations\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from framework.llm.key_pool import KeyPool\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n api_keys: list[str] | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n api_keys: Optional list of API keys for key-pool rotation. When\n provided with 2+ keys, a :class:`KeyPool` is created and\n keys are rotated on rate-limit errors.\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n # Key pool: when multiple keys are provided, enable rotation.\n self._key_pool: KeyPool | None = None\n if api_keys and len(api_keys) > 1:\n from framework.llm.key_pool import KeyPool\n\n self._key_pool = KeyPool(api_keys)\n self.api_key = api_keys[0] # default for OAuth detection below\n logger.info(\n \"[litellm] Key pool enabled with %d keys for model %s\",\n len(api_keys),\n model,\n )\n else:\n self.api_key = api_key or (api_keys[0] if api_keys else None)\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(self.api_key and self.api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\n\n When a :class:`KeyPool` is configured, rate-limited keys are rotated\n automatically so the next attempt uses a different key -- no sleep\n needed between attempts.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n # When we have other healthy keys, skip the sleep -- the\n # next iteration will pick a different key automatically.\n if attempt < retries:\n logger.info(\n \"[retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n When a :class:`KeyPool` is configured, rate-limited keys are rotated.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n # Rotate key from pool when available.\n current_key: str | None = None\n if self._key_pool:\n current_key = self._key_pool.get_key()\n kwargs[\"api_key\"] = current_key\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n if self._key_pool and current_key:\n self._key_pool.mark_success(current_key)\n return response\n except RateLimitError as e:\n # Key pool: mark the offending key and rotate immediately.\n if self._key_pool and current_key:\n self._key_pool.mark_rate_limited(current_key, retry_after=60.0)\n if attempt < retries:\n logger.info(\n \"[async-retry] Key pool rotating away from ...%s on 429\",\n current_key[-6:],\n )\n continue\n\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts -- rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n logger.info(\n \"[stream] extracted %d hallucinated tool call(s) from text\",\n len(extracted),\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "df41732e954019eca5e8df461a6f0d2978570b4e": {"t": 1785765557.3494742, "n": 90297, "h": "cfdbe9bbcd0693b1", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n tool_names = [tc.tool_name for tc in extracted]\n logger.info(\n \"[stream] Model emitted %d tool call(s) as <tool_code> text \"\n \"instead of structured function calls; converting to \"\n \"synthetic ToolCallEvents: %s\",\n len(extracted),\n tool_names,\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "ed8d417befcaec6b2c49205eae884687d5ff2b7c": {"t": 1785765558.9745815, "n": 90041, "h": "e1da3b327c5f6e21", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list, str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n key = f\"{tool_name}:{json.dumps(tool_args, sort_keys=True)}\"\n digest = hashlib.md5(key.encode()).hexdigest()[:12]\n call_id = f\"synth_{digest}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n logger.info(\n \"[stream] extracted %d hallucinated tool call(s) from text\",\n len(extracted),\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower() and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "294df7f06659f3b9f300359436d28e0d66923a97": {"t": 1785765560.5928674, "n": 90032, "h": "5c83ae48776ca04c", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list[\"ToolCallEvent\"], str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n call_id = f\"synth_{hashlib.md5(f'{tool_name}:{json.dumps(tool_args, sort_keys=True)}'.encode()).hexdigest()[:12]}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n logger.info(\n \"[stream] extracted %d hallucinated tool call(s) from text\",\n len(extracted),\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower()\n and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "b46fe697126675828de4268c444ebd7d1db05436": {"t": 1785765562.2956848, "n": 89838, "h": "a2bb9748530988a0", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list[\"ToolCallEvent\"], str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n call_id = f\"synth_{hashlib.md5(f'{tool_name}:{json.dumps(tool_args, sort_keys=True)}'.encode()).hexdigest()[:12]}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n # Kimi's 'pause_turn' means the model emitted tool\n # calls and expects results — equivalent to 'tool_calls'.\n if choice.finish_reason == \"pause_turn\":\n choice.finish_reason = \"tool_calls\" if tool_calls_acc else \"stop\"\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n logger.info(\n \"[stream] extracted %d hallucinated tool call(s) from text\",\n len(extracted),\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n # LiteLLM may wrap the original ValidationError in an\n # APIError with a different message. Check the full\n # exception chain (str(e) + str(__cause__)).\n _err_chain = f\"{e} {e.__cause__}\" if e.__cause__ else str(e)\n _is_finish_reason_err = (\n \"finish_reason\" in _err_chain and \"validation error\" in _err_chain.lower()\n ) or (\n # Fallback: the APIError wrapper message for chunk-building failures\n \"building chunks\" in str(e).lower()\n and (accumulated_text or tool_calls_acc)\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "934c4245103f4d18a02486498c02eb983eb204e1": {"t": 1785765565.1404703, "n": 88976, "h": "9f440c21d3d97368", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\ndef _extract_text_tool_calls(\n text: str,\n) -> tuple[list[\"ToolCallEvent\"], str]:\n \"\"\"Extract hallucinated tool calls from ``<tool_code>`` blocks in LLM text.\n\n Some models (notably Gemini) emit tool invocations as text instead of using\n the structured function-calling API. This function parses those blocks and\n returns ``(tool_call_events, cleaned_text)`` where *cleaned_text* has the\n ``<tool_code>`` blocks removed.\n\n Expected format::\n\n <tool_code>\n {\n \"tool_name\": { ...args }\n }\n </tool_code>\n \"\"\"\n from framework.llm.stream_events import ToolCallEvent\n\n pattern = re.compile(r\"<tool_code>\\s*(.*?)\\s*</tool_code>\", re.DOTALL)\n events: list[ToolCallEvent] = []\n cleaned = text\n\n for match in pattern.finditer(text):\n raw = match.group(1).strip()\n try:\n payload = json.loads(raw)\n except json.JSONDecodeError:\n logger.warning(\"[_extract_text_tool_calls] failed to parse JSON: %s\", raw[:200])\n continue\n\n if not isinstance(payload, dict):\n continue\n\n for tool_name, tool_args in payload.items():\n call_id = f\"synth_{hashlib.md5(f'{tool_name}:{json.dumps(tool_args, sort_keys=True)}'.encode()).hexdigest()[:12]}\"\n events.append(\n ToolCallEvent(\n tool_use_id=call_id,\n tool_name=tool_name,\n tool_input=tool_args if isinstance(tool_args, dict) else {},\n )\n )\n\n if events:\n cleaned = pattern.sub(\"\", text).strip()\n\n return events, cleaned\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Gemini sometimes outputs tool calls as text in\n # <tool_code>{\"name\": {...args}}</tool_code> blocks\n # instead of using the function-calling API. Extract\n # these as real ToolCallEvents and strip them from the\n # text so the rest of the system treats them normally.\n if accumulated_text and \"<tool_code>\" in accumulated_text:\n extracted, cleaned = _extract_text_tool_calls(accumulated_text)\n if extracted:\n logger.info(\n \"[stream] extracted %d hallucinated tool call(s) from text\",\n len(extracted),\n )\n accumulated_text = cleaned\n # Emit a corrected TextDeltaEvent so the caller's\n # accumulated_text is overwritten with the cleaned text.\n yield TextDeltaEvent(content=\"\", snapshot=cleaned)\n # Insert synthetic ToolCallEvents before FinishEvent.\n finish_idx = next(\n (i for i, ev in enumerate(tail_events) if isinstance(ev, FinishEvent)),\n len(tail_events),\n )\n for tc_ev in reversed(extracted):\n tail_events.insert(finish_idx, tc_ev)\n # Update TextEndEvent if present.\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, TextEndEvent):\n tail_events[_i] = TextEndEvent(full_text=cleaned)\n break\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n # Some providers return non-standard finish_reason values\n # (e.g., kimi-k2.5 sends 'pause_turn') that LiteLLM's\n # internal stream_chunk_builder rejects via Pydantic\n # validation. If we already accumulated content and built\n # tail_events before the error, the stream was successful —\n # yield what we have instead of discarding it.\n if (accumulated_text or tool_calls_acc) and tail_events:\n _is_finish_reason_err = (\n \"finish_reason\" in str(e) and \"validation error\" in str(e).lower()\n )\n if _is_finish_reason_err:\n logger.warning(\n \"[stream] %s: LiteLLM finish_reason validation \"\n \"error (non-standard provider value). \"\n \"Content was streamed successfully — \"\n \"using accumulated result. Error: %s\",\n self.model,\n e,\n )\n for event in tail_events:\n yield event\n return\n\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}, "4d877469d5af3a557316485863da9a7c4217e169": {"t": 1785765568.0567532, "n": 84560, "h": "537d5ded1c2287cd", "c": "\"\"\"LiteLLM provider for pluggable multi-provider LLM support.\n\nLiteLLM provides a unified, OpenAI-compatible interface that supports\nmultiple LLM providers including OpenAI, Anthropic, Gemini, Mistral,\nGroq, and local models.\n\nSee: https://docs.litellm.ai/docs/providers\n\"\"\"\n\nimport ast\nimport asyncio\nimport hashlib\nimport json\nimport logging\nimport os\nimport re\nimport time\nfrom collections.abc import AsyncIterator\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\ntry:\n import litellm\n from litellm.exceptions import RateLimitError\nexcept ImportError:\n litellm = None # type: ignore[assignment]\n RateLimitError = Exception # type: ignore[assignment, misc]\n\nfrom framework.config import HIVE_LLM_ENDPOINT as HIVE_API_BASE\nfrom framework.llm.provider import LLMProvider, LLMResponse, Tool\nfrom framework.llm.stream_events import StreamEvent\n\nlogger = logging.getLogger(__name__)\n\n\ndef _patch_litellm_anthropic_oauth() -> None:\n \"\"\"Patch litellm's Anthropic header construction to fix OAuth token handling.\n\n litellm bug: validate_environment() puts the OAuth token into x-api-key,\n but Anthropic's API rejects OAuth tokens in x-api-key. They must be sent\n via Authorization: Bearer only, with x-api-key omitted entirely.\n\n This patch wraps validate_environment to remove x-api-key when the\n Authorization header carries an OAuth token (sk-ant-oat prefix).\n\n See: https://github.com/BerriAI/litellm/issues/19618\n \"\"\"\n try:\n from litellm.llms.anthropic.common_utils import AnthropicModelInfo\n from litellm.types.llms.anthropic import (\n ANTHROPIC_OAUTH_BETA_HEADER,\n ANTHROPIC_OAUTH_TOKEN_PREFIX,\n )\n except ImportError:\n logger.warning(\n \"Could not apply litellm Anthropic OAuth patch — litellm internals may have \"\n \"changed. Anthropic OAuth tokens (Claude Code subscriptions) may fail with 401. \"\n \"See BerriAI/litellm#19618. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n return\n\n original = AnthropicModelInfo.validate_environment\n\n def _patched_validate_environment(\n self, headers, model, messages, optional_params, litellm_params, api_key=None, api_base=None\n ):\n result = original(\n self,\n headers,\n model,\n messages,\n optional_params,\n litellm_params,\n api_key=api_key,\n api_base=api_base,\n )\n # Check both authorization header and x-api-key for OAuth tokens.\n # litellm's optionally_handle_anthropic_oauth only checks headers[\"authorization\"],\n # but hive passes OAuth tokens via api_key — so litellm puts them into x-api-key.\n # Anthropic rejects OAuth tokens in x-api-key; they must go in Authorization: Bearer.\n auth = result.get(\"authorization\", \"\")\n x_api_key = result.get(\"x-api-key\", \"\")\n oauth_prefix = f\"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}\"\n auth_is_oauth = auth.startswith(oauth_prefix)\n key_is_oauth = x_api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)\n if auth_is_oauth or key_is_oauth:\n token = x_api_key if key_is_oauth else auth.removeprefix(\"Bearer \").strip()\n result.pop(\"x-api-key\", None)\n result[\"authorization\"] = f\"Bearer {token}\"\n # Merge the OAuth beta header with any existing beta headers.\n existing_beta = result.get(\"anthropic-beta\", \"\")\n beta_parts = (\n [b.strip() for b in existing_beta.split(\",\") if b.strip()] if existing_beta else []\n )\n if ANTHROPIC_OAUTH_BETA_HEADER not in beta_parts:\n beta_parts.append(ANTHROPIC_OAUTH_BETA_HEADER)\n result[\"anthropic-beta\"] = \",\".join(beta_parts)\n return result\n\n AnthropicModelInfo.validate_environment = _patched_validate_environment\n\n\ndef _patch_litellm_metadata_nonetype() -> None:\n \"\"\"Patch litellm entry points to prevent metadata=None TypeError.\n\n litellm bug: the @client decorator in utils.py has four places that do\n \"model_group\" in kwargs.get(\"metadata\", {})\n but kwargs[\"metadata\"] can be explicitly None (set internally by\n litellm_params), causing:\n TypeError: argument of type 'NoneType' is not iterable\n This masks the real API error with a confusing APIConnectionError.\n\n Fix: wrap the four litellm entry points (completion, acompletion,\n responses, aresponses) to pop metadata=None before the @client\n decorator's error handler can crash on it.\n \"\"\"\n import functools\n\n patched_count = 0\n for fn_name in (\"completion\", \"acompletion\", \"responses\", \"aresponses\"):\n original = getattr(litellm, fn_name, None)\n if original is None:\n continue\n patched_count += 1\n if asyncio.iscoroutinefunction(original):\n\n @functools.wraps(original)\n async def _async_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return await _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _async_wrapper)\n else:\n\n @functools.wraps(original)\n def _sync_wrapper(*args, _orig=original, **kwargs):\n if kwargs.get(\"metadata\") is None:\n kwargs.pop(\"metadata\", None)\n return _orig(*args, **kwargs)\n\n setattr(litellm, fn_name, _sync_wrapper)\n\n if patched_count == 0:\n logger.warning(\n \"Could not apply litellm metadata=None patch — none of the expected entry \"\n \"points (completion, acompletion, responses, aresponses) were found. \"\n \"metadata=None TypeError may occur. Current litellm version: %s\",\n getattr(litellm, \"__version__\", \"unknown\"),\n )\n\n\nif litellm is not None:\n _patch_litellm_anthropic_oauth()\n _patch_litellm_metadata_nonetype()\n # Let litellm silently drop params unsupported by the target provider\n # (e.g. stream_options for Anthropic) instead of forwarding them verbatim.\n litellm.drop_params = True\n\n\ndef _is_ollama_model(model: str) -> bool:\n \"\"\"Return True for any Ollama model string (ollama/ or ollama_chat/ prefix).\"\"\"\n return model.startswith(\"ollama/\") or model.startswith(\"ollama_chat/\")\n\n\ndef _ensure_ollama_chat_prefix(model: str) -> str:\n \"\"\"Normalise Ollama model strings to use the ollama_chat/ prefix.\n\n LiteLLM requires the ``ollama_chat/`` prefix (not ``ollama/``) to enable\n native function-calling support. With ``ollama/``, LiteLLM falls back to\n JSON-mode tool calls, which the framework cannot parse as real tool calls.\n\n See: https://docs.litellm.ai/docs/providers/ollama#example-usage---tool-calling\n \"\"\"\n if model.startswith(\"ollama/\"):\n return \"ollama_chat/\" + model[len(\"ollama/\") :]\n return model\n\n\nRATE_LIMIT_MAX_RETRIES = 10\nRATE_LIMIT_BACKOFF_BASE = 2 # seconds\nRATE_LIMIT_MAX_DELAY = 120 # seconds - cap to prevent absurd waits\nMINIMAX_API_BASE = \"https://api.minimax.io/v1\"\nOPENROUTER_API_BASE = \"https://openrouter.ai/api/v1\"\n\n# Providers that accept cache_control on message content blocks.\n# Anthropic: native ephemeral caching. MiniMax & Z-AI/GLM: pass-through to their APIs.\n# (OpenAI caches automatically server-side; Groq/Gemini/etc. strip the header.)\n_CACHE_CONTROL_PREFIXES = (\n \"anthropic/\",\n \"claude-\",\n \"minimax/\",\n \"minimax-\",\n \"MiniMax-\",\n \"zai-glm\",\n \"glm-\",\n)\n\n\ndef _model_supports_cache_control(model: str) -> bool:\n return any(model.startswith(p) for p in _CACHE_CONTROL_PREFIXES)\n\n\n# Kimi For Coding uses an Anthropic-compatible endpoint (no /v1 suffix).\n# Claude Code integration uses this format; the /v1 OpenAI-compatible endpoint\n# enforces a coding-agent whitelist that blocks unknown User-Agents.\nKIMI_API_BASE = \"https://api.kimi.com/coding\"\n\n# Claude Code OAuth subscription: the Anthropic API requires a specific\n# User-Agent and a billing integrity header for OAuth-authenticated requests.\nCLAUDE_CODE_VERSION = \"2.1.76\"\nCLAUDE_CODE_USER_AGENT = f\"claude-code/{CLAUDE_CODE_VERSION}\"\n_CLAUDE_CODE_BILLING_SALT = \"59cf53e54c78\"\n\n\ndef _sample_js_code_unit(text: str, idx: int) -> str:\n \"\"\"Return the character at UTF-16 code unit index *idx*, matching JS semantics.\"\"\"\n encoded = text.encode(\"utf-16-le\")\n unit_offset = idx * 2\n if unit_offset + 2 > len(encoded):\n return \"0\"\n code_unit = int.from_bytes(encoded[unit_offset : unit_offset + 2], \"little\")\n return chr(code_unit)\n\n\ndef _claude_code_billing_header(messages: list[dict[str, Any]]) -> str:\n \"\"\"Build the billing integrity system block required by Anthropic's OAuth path.\"\"\"\n # Find the first user message text\n first_text = \"\"\n for msg in messages:\n if msg.get(\"role\") != \"user\":\n continue\n content = msg.get(\"content\")\n if isinstance(content, str):\n first_text = content\n break\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\" and block.get(\"text\"):\n first_text = block[\"text\"]\n break\n if first_text:\n break\n\n sampled = \"\".join(_sample_js_code_unit(first_text, i) for i in (4, 7, 20))\n version_hash = hashlib.sha256(\n f\"{_CLAUDE_CODE_BILLING_SALT}{sampled}{CLAUDE_CODE_VERSION}\".encode()\n ).hexdigest()\n entrypoint = os.environ.get(\"CLAUDE_CODE_ENTRYPOINT\", \"\").strip() or \"cli\"\n return (\n f\"x-anthropic-billing-header: cc_version={CLAUDE_CODE_VERSION}.{version_hash[:3]}; \"\n f\"cc_entrypoint={entrypoint}; cch=00000;\"\n )\n\n\n# Empty-stream retries use a short fixed delay, not the rate-limit backoff.\n# Conversation-structure issues are deterministic — long waits don't help.\nEMPTY_STREAM_MAX_RETRIES = 3\nEMPTY_STREAM_RETRY_DELAY = 1.0 # seconds\nOPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS = (\n \"no endpoints found that support tool use\",\n \"no endpoints available that support tool use\",\n \"provider routing\",\n)\nOPENROUTER_TOOL_CALL_RE = re.compile(\n r\"<\\|tool_call_start\\|>\\s*(.*?)\\s*<\\|tool_call_end\\|>\",\n re.DOTALL,\n)\nOPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS = 3600\n# OpenRouter routing can change over time, so tool-compat caching must expire.\nOPENROUTER_TOOL_COMPAT_MODEL_CACHE: dict[str, float] = {}\n\n# Transient stream errors (network blips, timeouts) use a separate cap\n# from rate-limit retries — 3 retries is sufficient for connection failures.\nSTREAM_TRANSIENT_MAX_RETRIES = 3\n\n# Directory for dumping failed requests\nFAILED_REQUESTS_DIR = Path.home() / \".hive\" / \"failed_requests\"\n\n# Maximum number of dump files to retain in ~/.hive/failed_requests/.\n# Older files are pruned automatically to prevent unbounded disk growth.\nMAX_FAILED_REQUEST_DUMPS = 50\n\n\ndef _estimate_tokens(model: str, messages: list[dict]) -> tuple[int, str]:\n \"\"\"Estimate token count for messages. Returns (token_count, method).\"\"\"\n # Try litellm's token counter first\n if litellm is not None:\n try:\n count = litellm.token_counter(model=model, messages=messages)\n return count, \"litellm\"\n except Exception:\n pass\n\n # Fallback: rough estimate based on character count (~4 chars per token)\n total_chars = sum(len(str(m.get(\"content\", \"\"))) for m in messages)\n return total_chars // 4, \"estimate\"\n\n\ndef _prune_failed_request_dumps(max_files: int = MAX_FAILED_REQUEST_DUMPS) -> None:\n \"\"\"Remove oldest dump files when the count exceeds *max_files*.\n\n Best-effort: never raises — a pruning failure must not break retry logic.\n \"\"\"\n try:\n all_dumps = sorted(\n FAILED_REQUESTS_DIR.glob(\"*.json\"),\n key=lambda f: f.stat().st_mtime,\n )\n excess = len(all_dumps) - max_files\n if excess > 0:\n for old_file in all_dumps[:excess]:\n old_file.unlink(missing_ok=True)\n except Exception:\n pass # Best-effort — never block the caller\n\n\ndef _remember_openrouter_tool_compat_model(model: str) -> None:\n \"\"\"Cache OpenRouter tool-compat fallback for a bounded time window.\"\"\"\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE[model] = (\n time.monotonic() + OPENROUTER_TOOL_COMPAT_CACHE_TTL_SECONDS\n )\n\n\ndef _is_openrouter_tool_compat_cached(model: str) -> bool:\n \"\"\"Return True when the cached OpenRouter compat entry is still fresh.\"\"\"\n expires_at = OPENROUTER_TOOL_COMPAT_MODEL_CACHE.get(model)\n if expires_at is None:\n return False\n if expires_at <= time.monotonic():\n OPENROUTER_TOOL_COMPAT_MODEL_CACHE.pop(model, None)\n return False\n return True\n\n\ndef _dump_failed_request(\n model: str,\n kwargs: dict[str, Any],\n error_type: str,\n attempt: int,\n) -> str:\n \"\"\"Dump failed request to a file for debugging. Returns the file path.\"\"\"\n FAILED_REQUESTS_DIR.mkdir(parents=True, exist_ok=True)\n\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S_%f\")\n filename = f\"{error_type}_{model.replace('/', '_')}_{timestamp}.json\"\n filepath = FAILED_REQUESTS_DIR / filename\n\n # Build dump data\n messages = kwargs.get(\"messages\", [])\n dump_data = {\n \"timestamp\": datetime.now().isoformat(),\n \"model\": model,\n \"error_type\": error_type,\n \"attempt\": attempt,\n \"estimated_tokens\": _estimate_tokens(model, messages),\n \"num_messages\": len(messages),\n \"messages\": messages,\n \"tools\": kwargs.get(\"tools\"),\n \"max_tokens\": kwargs.get(\"max_tokens\"),\n \"temperature\": kwargs.get(\"temperature\"),\n }\n\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n json.dump(dump_data, f, indent=2, default=str)\n\n # Prune old dumps to prevent unbounded disk growth\n _prune_failed_request_dumps()\n\n return str(filepath)\n\n\ndef _compute_retry_delay(\n attempt: int,\n exception: BaseException | None = None,\n backoff_base: int = RATE_LIMIT_BACKOFF_BASE,\n max_delay: int = RATE_LIMIT_MAX_DELAY,\n) -> float:\n \"\"\"Compute retry delay, preferring server-provided Retry-After headers.\n\n Priority:\n 1. retry-after-ms header (milliseconds, float)\n 2. retry-after header as seconds (float)\n 3. retry-after header as HTTP-date (RFC 7231)\n 4. Exponential backoff: backoff_base * 2^attempt\n\n All values are capped at max_delay seconds.\n \"\"\"\n if exception is not None:\n response = getattr(exception, \"response\", None)\n if response is not None:\n headers = getattr(response, \"headers\", None)\n if headers is not None:\n # Priority 1: retry-after-ms (milliseconds)\n retry_after_ms = headers.get(\"retry-after-ms\")\n if retry_after_ms is not None:\n try:\n delay = float(retry_after_ms) / 1000.0\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Priority 2: retry-after (seconds or HTTP-date)\n retry_after = headers.get(\"retry-after\")\n if retry_after is not None:\n # Try as seconds (float)\n try:\n delay = float(retry_after)\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError):\n pass\n\n # Try as HTTP-date (e.g., \"Fri, 31 Dec 2025 23:59:59 GMT\")\n try:\n from email.utils import parsedate_to_datetime\n\n retry_date = parsedate_to_datetime(retry_after)\n now = datetime.now(retry_date.tzinfo)\n delay = (retry_date - now).total_seconds()\n return min(max(delay, 0), max_delay)\n except (ValueError, TypeError, OverflowError):\n pass\n\n # Fallback: exponential backoff\n delay = backoff_base * (2**attempt)\n return min(delay, max_delay)\n\n\ndef _is_stream_transient_error(exc: BaseException) -> bool:\n \"\"\"Classify whether a streaming exception is transient (recoverable).\n\n Transient errors (recoverable=True): network issues, server errors, timeouts.\n Permanent errors (recoverable=False): auth, bad request, context window, etc.\n\n NOTE: \"Failed to parse tool call arguments\" (malformed LLM output) is NOT\n transient at the stream level — retrying with the same messages produces the\n same malformed output. This error is handled at the EventLoopNode level\n where the conversation can be modified before retrying.\n \"\"\"\n try:\n from litellm.exceptions import (\n APIConnectionError,\n BadGatewayError,\n InternalServerError,\n ServiceUnavailableError,\n )\n\n transient_types: tuple[type[BaseException], ...] = (\n APIConnectionError,\n InternalServerError,\n BadGatewayError,\n ServiceUnavailableError,\n TimeoutError,\n ConnectionError,\n OSError,\n )\n except ImportError:\n transient_types = (TimeoutError, ConnectionError, OSError)\n\n return isinstance(exc, transient_types)\n\n\nclass LiteLLMProvider(LLMProvider):\n \"\"\"\n LiteLLM-based LLM provider for multi-provider support.\n\n Supports any model that LiteLLM supports, including:\n - OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo\n - Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku\n - Google: gemini-pro, gemini-1.5-pro, gemini-1.5-flash\n - DeepSeek: deepseek-chat, deepseek-coder, deepseek-reasoner\n - Mistral: mistral-large, mistral-medium, mistral-small\n - Groq: llama3-70b, mixtral-8x7b\n - Local: ollama/llama3, ollama/mistral\n - And many more...\n\n Usage:\n # OpenAI\n provider = LiteLLMProvider(model=\"gpt-4o-mini\")\n\n # Anthropic\n provider = LiteLLMProvider(model=\"claude-3-haiku-20240307\")\n\n # Google Gemini\n provider = LiteLLMProvider(model=\"gemini/gemini-1.5-flash\")\n\n # DeepSeek\n provider = LiteLLMProvider(model=\"deepseek/deepseek-chat\")\n\n # Local Ollama\n provider = LiteLLMProvider(model=\"ollama/llama3\")\n\n # With custom API base\n provider = LiteLLMProvider(\n model=\"gpt-4o-mini\",\n api_base=\"https://my-proxy.com/v1\"\n )\n \"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o-mini\",\n api_key: str | None = None,\n api_base: str | None = None,\n **kwargs: Any,\n ):\n \"\"\"\n Initialize the LiteLLM provider.\n\n Args:\n model: Model identifier (e.g., \"gpt-4o-mini\", \"claude-3-haiku-20240307\")\n LiteLLM auto-detects the provider from the model name.\n api_key: API key for the provider. If not provided, LiteLLM will\n look for the appropriate env var (OPENAI_API_KEY,\n ANTHROPIC_API_KEY, etc.)\n api_base: Custom API base URL (for proxies or local deployments)\n **kwargs: Additional arguments passed to litellm.completion()\n \"\"\"\n # Kimi For Coding exposes an Anthropic-compatible endpoint at\n # https://api.kimi.com/coding (the same format Claude Code uses natively).\n # Translate kimi/ prefix to anthropic/ so litellm uses the Anthropic\n # Messages API handler and routes to that endpoint — no special headers needed.\n _original_model = model\n if _is_ollama_model(model):\n model = _ensure_ollama_chat_prefix(model)\n elif model.lower().startswith(\"kimi/\"):\n model = \"anthropic/\" + model[len(\"kimi/\") :]\n # Normalise api_base: litellm's Anthropic handler appends /v1/messages,\n # so the base must be https://api.kimi.com/coding (no /v1 suffix).\n # Strip a trailing /v1 in case the user's saved config has the old value.\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n elif model.lower().startswith(\"hive/\"):\n model = \"anthropic/\" + model[len(\"hive/\") :]\n if api_base and api_base.rstrip(\"/\").endswith(\"/v1\"):\n api_base = api_base.rstrip(\"/\")[:-3]\n self.model = model\n self.api_key = api_key\n self.api_base = api_base or self._default_api_base_for_model(_original_model)\n self.extra_kwargs = kwargs\n # Detect Claude Code OAuth subscription by checking the api_key prefix.\n self._claude_code_oauth = bool(api_key and api_key.startswith(\"sk-ant-oat\"))\n if self._claude_code_oauth:\n # Anthropic requires a specific User-Agent for OAuth requests.\n eh = self.extra_kwargs.setdefault(\"extra_headers\", {})\n eh.setdefault(\"user-agent\", CLAUDE_CODE_USER_AGENT)\n # The Codex ChatGPT backend (chatgpt.com/backend-api/codex) rejects\n # several standard OpenAI params: max_output_tokens, stream_options.\n self._codex_backend = bool(\n self.api_base and \"chatgpt.com/backend-api/codex\" in self.api_base\n )\n # Antigravity routes through a local OpenAI-compatible proxy — no patches needed.\n self._antigravity = bool(self.api_base and \"localhost:8069\" in self.api_base)\n\n if litellm is None:\n raise ImportError(\n \"LiteLLM is not installed. Please install it with: uv pip install litellm\"\n )\n\n # Note: The Codex ChatGPT backend is a Responses API endpoint at\n # chatgpt.com/backend-api/codex/responses. LiteLLM's model registry\n # correctly marks codex models with mode=\"responses\", so we do NOT\n # override the mode. The responses_api_bridge in litellm handles\n # converting Chat Completions requests to Responses API format.\n\n @staticmethod\n def _default_api_base_for_model(model: str) -> str | None:\n \"\"\"Return provider-specific default API base when required.\"\"\"\n model_lower = model.lower()\n if model_lower.startswith(\"minimax/\") or model_lower.startswith(\"minimax-\"):\n return MINIMAX_API_BASE\n if model_lower.startswith(\"openrouter/\"):\n return OPENROUTER_API_BASE\n if model_lower.startswith(\"kimi/\"):\n return KIMI_API_BASE\n if model_lower.startswith(\"hive/\"):\n return HIVE_API_BASE\n return None\n\n def _completion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Call litellm.completion with retry on 429 rate limit errors and empty responses.\"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = litellm.completion(**kwargs) # type: ignore[union-attr]\n\n # Some providers (e.g. Gemini) return 200 with empty content on\n # rate limit / quota exhaustion instead of a proper 429. Treat\n # empty responses the same as a rate-limit error and retry.\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n # If the conversation ends with an assistant message,\n # an empty response is expected — don't retry.\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n # Dump full request to file for debugging\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n # Dump full request to file for debugging\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n time.sleep(wait)\n # unreachable, but satisfies type checker\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n def complete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Generate a completion using LiteLLM.\"\"\"\n # Codex ChatGPT backend requires streaming — delegate to the unified\n # async streaming path which properly handles tool calls.\n if self._codex_backend:\n return asyncio.run(\n self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n max_retries=max_retries,\n )\n )\n\n # Prepare messages with system prompt\n full_messages = []\n if system:\n full_messages.append({\"role\": \"system\", \"content\": system})\n full_messages.extend(messages)\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n # Append to system message if present, otherwise add as system message\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Build kwargs\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n # Add tools if provided\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n\n # Add response_format for structured output\n # LiteLLM passes this through to the underlying provider\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n # Make the call\n response = self._completion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n # Extract content\n content = response.choices[0].message.content or \"\"\n\n # Get usage info.\n # NOTE: completion_tokens includes reasoning/thinking tokens for models\n # that use them (o1, gpt-5-mini, etc.). LiteLLM does not reliably expose\n # usage.completion_tokens_details.reasoning_tokens across all providers.\n # This means output_tokens may be inflated for reasoning models.\n # Compaction is unaffected — it uses prompt_tokens (input-side only).\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n # ------------------------------------------------------------------\n # Async variants — non-blocking on the event loop\n # ------------------------------------------------------------------\n\n async def _acompletion_with_rate_limit_retry(\n self, max_retries: int | None = None, **kwargs: Any\n ) -> Any:\n \"\"\"Async version of _completion_with_rate_limit_retry.\n\n Uses litellm.acompletion and asyncio.sleep instead of blocking calls.\n \"\"\"\n model = kwargs.get(\"model\", self.model)\n retries = max_retries if max_retries is not None else RATE_LIMIT_MAX_RETRIES\n for attempt in range(retries + 1):\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n content = response.choices[0].message.content if response.choices else None\n has_tool_calls = bool(response.choices and response.choices[0].message.tool_calls)\n if not content and not has_tool_calls:\n messages = kwargs.get(\"messages\", [])\n last_role = next(\n (m[\"role\"] for m in reversed(messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if last_role == \"assistant\":\n logger.debug(\n \"[async-retry] Empty response after assistant message — \"\n \"expected, not retrying.\"\n )\n return response\n\n finish_reason = (\n response.choices[0].finish_reason if response.choices else \"unknown\"\n )\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"empty_response\",\n attempt=attempt,\n )\n logger.warning(\n f\"[async-retry] Empty response - {len(messages)} messages, \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n\n # finish_reason=length means the model exhausted max_tokens\n # before producing content. Retrying with the same max_tokens\n # will never help — return immediately instead of looping.\n if finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[async-retry] {model} returned empty content with \"\n f\"finish_reason=length (max_tokens={max_tok}). \"\n f\"The model exhausted its token budget before \"\n f\"producing visible output. Increase max_tokens \"\n f\"or use a different model. Not retrying.\"\n )\n return response\n\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0})\"\n )\n return response\n wait = _compute_retry_delay(attempt)\n logger.warning(\n f\"[async-retry] {model} returned empty response \"\n f\"(finish_reason={finish_reason}, \"\n f\"choices={len(response.choices) if response.choices else 0}) — \"\n f\"likely rate limited or quota exceeded. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n continue\n\n return response\n except RateLimitError as e:\n messages = kwargs.get(\"messages\", [])\n token_count, token_method = _estimate_tokens(model, messages)\n dump_path = _dump_failed_request(\n model=model,\n kwargs=kwargs,\n error_type=\"rate_limit\",\n attempt=attempt,\n )\n if attempt == retries:\n logger.error(\n f\"[async-retry] GAVE UP on {model} after {retries + 1} \"\n f\"attempts — rate limit error: {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}\"\n )\n raise\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[async-retry] {model} rate limited (429): {e!s}. \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Full request dumped to: {dump_path}. \"\n f\"Retrying in {wait}s \"\n f\"(attempt {attempt + 1}/{retries})\"\n )\n await asyncio.sleep(wait)\n raise RuntimeError(\"Exhausted rate limit retries\")\n\n async def acomplete(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 1024,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n max_retries: int | None = None,\n ) -> LLMResponse:\n \"\"\"Async version of complete(). Uses litellm.acompletion — non-blocking.\"\"\"\n # Codex ChatGPT backend requires streaming — route through stream() which\n # already handles Codex quirks and has proper tool call accumulation.\n if self._codex_backend:\n stream_iter = self.stream(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n return await self._collect_stream_to_response(stream_iter)\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n\n response = await self._acompletion_with_rate_limit_retry(max_retries=max_retries, **kwargs)\n\n content = response.choices[0].message.content or \"\"\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n\n return LLMResponse(\n content=content,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=response.choices[0].finish_reason or \"\",\n raw_response=response,\n )\n\n def _tool_to_openai_format(self, tool: Tool) -> dict[str, Any]:\n \"\"\"Convert Tool to OpenAI function calling format.\"\"\"\n return {\n \"type\": \"function\",\n \"function\": {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": tool.parameters.get(\"properties\", {}),\n \"required\": tool.parameters.get(\"required\", []),\n },\n },\n }\n\n def _is_anthropic_model(self) -> bool:\n \"\"\"Return True when the configured model targets Anthropic.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"anthropic/\") or model.startswith(\"claude-\")\n\n def _is_minimax_model(self) -> bool:\n \"\"\"Return True when the configured model targets MiniMax.\"\"\"\n model = (self.model or \"\").lower()\n return model.startswith(\"minimax/\") or model.startswith(\"minimax-\")\n\n def _is_openrouter_model(self) -> bool:\n \"\"\"Return True when the configured model targets OpenRouter.\"\"\"\n model = (self.model or \"\").lower()\n if model.startswith(\"openrouter/\"):\n return True\n api_base = (self.api_base or \"\").lower()\n return \"openrouter.ai/api/v1\" in api_base\n\n def _should_use_openrouter_tool_compat(\n self,\n error: BaseException,\n tools: list[Tool] | None,\n ) -> bool:\n \"\"\"Return True when OpenRouter rejects native tool use for the model.\"\"\"\n if not tools or not self._is_openrouter_model():\n return False\n error_text = str(error).lower()\n return \"openrouter\" in error_text and any(\n snippet in error_text for snippet in OPENROUTER_TOOL_COMPAT_ERROR_SNIPPETS\n )\n\n @staticmethod\n def _extract_json_object(text: str) -> dict[str, Any] | None:\n \"\"\"Extract the first JSON object from a model response.\"\"\"\n candidates = [text.strip()]\n\n stripped = text.strip()\n if stripped.startswith(\"```\"):\n fence_lines = stripped.splitlines()\n if len(fence_lines) >= 3:\n candidates.append(\"\\n\".join(fence_lines[1:-1]).strip())\n\n decoder = json.JSONDecoder()\n for candidate in candidates:\n if not candidate:\n continue\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n parsed = None\n if isinstance(parsed, dict):\n return parsed\n\n for start_idx, char in enumerate(candidate):\n if char != \"{\":\n continue\n try:\n parsed, _ = decoder.raw_decode(candidate[start_idx:])\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_openrouter_tool_compat_response(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse JSON tool-compat output into assistant text and tool calls.\"\"\"\n payload = self._extract_json_object(content)\n if payload is None:\n text_tool_content, text_tool_calls = self._parse_openrouter_text_tool_calls(\n content,\n tools,\n )\n if text_tool_calls:\n logger.info(\n \"[openrouter-tool-compat] Parsed textual tool-call markers for %s\",\n self.model,\n )\n return text_tool_content, text_tool_calls\n logger.info(\n \"[openrouter-tool-compat] %s returned non-JSON fallback content; \"\n \"treating it as plain text.\",\n self.model,\n )\n return content.strip(), []\n\n assistant_text = payload.get(\"assistant_response\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"content\")\n if not isinstance(assistant_text, str):\n assistant_text = payload.get(\"response\")\n if not isinstance(assistant_text, str):\n assistant_text = \"\"\n\n tool_calls_raw = payload.get(\"tool_calls\")\n if not tool_calls_raw and {\"name\", \"arguments\"} <= payload.keys():\n tool_calls_raw = [payload]\n elif isinstance(payload.get(\"tool_call\"), dict):\n tool_calls_raw = [payload[\"tool_call\"]]\n\n if not isinstance(tool_calls_raw, list):\n tool_calls_raw = []\n\n allowed_tool_names = {tool.name for tool in tools}\n tool_calls: list[dict[str, Any]] = []\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n\n for idx, raw_call in enumerate(tool_calls_raw):\n if not isinstance(raw_call, dict):\n continue\n\n function_block = raw_call.get(\"function\")\n function_name = (\n raw_call.get(\"name\")\n or raw_call.get(\"tool_name\")\n or (function_block.get(\"name\") if isinstance(function_block, dict) else None)\n )\n if not isinstance(function_name, str) or function_name not in allowed_tool_names:\n if function_name:\n logger.warning(\n \"[openrouter-tool-compat] Ignoring unknown tool '%s' for model %s\",\n function_name,\n self.model,\n )\n continue\n\n arguments = raw_call.get(\"arguments\")\n if arguments is None:\n arguments = raw_call.get(\"tool_input\")\n if arguments is None:\n arguments = raw_call.get(\"input\")\n if arguments is None and isinstance(function_block, dict):\n arguments = function_block.get(\"arguments\")\n if arguments is None:\n arguments = {}\n\n if isinstance(arguments, str):\n try:\n arguments = json.loads(arguments)\n except json.JSONDecodeError:\n arguments = {\"_raw\": arguments}\n elif not isinstance(arguments, dict):\n arguments = {\"value\": arguments}\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{idx}\",\n \"name\": function_name,\n \"input\": arguments,\n }\n )\n\n return assistant_text.strip(), tool_calls\n\n @staticmethod\n def _close_truncated_json_fragment(fragment: str) -> str:\n \"\"\"Close a truncated JSON fragment by balancing quotes/brackets.\"\"\"\n stack: list[str] = []\n in_string = False\n escaped = False\n normalized = fragment.rstrip()\n\n while normalized and normalized[-1] in \",:{[\":\n normalized = normalized[:-1].rstrip()\n\n for char in normalized:\n if in_string:\n if escaped:\n escaped = False\n elif char == \"\\\\\":\n escaped = True\n elif char == '\"':\n in_string = False\n continue\n\n if char == '\"':\n in_string = True\n elif char in \"{[\":\n stack.append(char)\n elif char == \"}\" and stack and stack[-1] == \"{\":\n stack.pop()\n elif char == \"]\" and stack and stack[-1] == \"[\":\n stack.pop()\n\n if in_string:\n if escaped:\n normalized = normalized[:-1]\n normalized += '\"'\n\n for opener in reversed(stack):\n normalized += \"}\" if opener == \"{\" else \"]\"\n\n return normalized\n\n def _repair_truncated_tool_arguments(self, raw_arguments: str) -> dict[str, Any] | None:\n \"\"\"Try to recover a truncated JSON object from tool-call arguments.\"\"\"\n stripped = raw_arguments.strip()\n if not stripped or stripped[0] != \"{\":\n return None\n\n max_trim = min(len(stripped), 256)\n for trim in range(max_trim + 1):\n candidate = stripped[: len(stripped) - trim].rstrip()\n if not candidate:\n break\n candidate = self._close_truncated_json_fragment(candidate)\n try:\n parsed = json.loads(candidate)\n except json.JSONDecodeError:\n continue\n if isinstance(parsed, dict):\n return parsed\n return None\n\n def _parse_tool_call_arguments(self, raw_arguments: str, tool_name: str) -> dict[str, Any]:\n \"\"\"Parse streamed tool arguments, repairing truncation when possible.\"\"\"\n try:\n parsed = json.loads(raw_arguments) if raw_arguments else {}\n except json.JSONDecodeError:\n parsed = None\n\n if isinstance(parsed, dict):\n return parsed\n\n repaired = self._repair_truncated_tool_arguments(raw_arguments)\n if repaired is not None:\n logger.warning(\n \"[tool-args] Recovered truncated arguments for %s on %s\",\n tool_name,\n self.model,\n )\n return repaired\n\n raise ValueError(\n f\"Failed to parse tool call arguments for '{tool_name}' (likely truncated JSON).\"\n )\n\n def _parse_openrouter_text_tool_calls(\n self,\n content: str,\n tools: list[Tool],\n ) -> tuple[str, list[dict[str, Any]]]:\n \"\"\"Parse textual OpenRouter tool calls into synthetic tool calls.\n\n Supports both:\n - Marker wrapped payloads: <|tool_call_start|>...<|tool_call_end|>\n - Plain one-line tool calls: ask_user(\"...\", [\"...\"])\n \"\"\"\n tools_by_name = {tool.name: tool for tool in tools}\n compat_prefix = f\"openrouter_compat_{time.time_ns()}\"\n tool_calls: list[dict[str, Any]] = []\n segment_index = 0\n\n for match in OPENROUTER_TOOL_CALL_RE.finditer(content):\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=match.group(1),\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n\n stripped_content = OPENROUTER_TOOL_CALL_RE.sub(\"\", content)\n retained_lines: list[str] = []\n for line in stripped_content.splitlines():\n stripped_line = line.strip()\n if not stripped_line:\n retained_lines.append(line)\n continue\n\n candidate = stripped_line\n if candidate.startswith(\"`\") and candidate.endswith(\"`\") and len(candidate) > 1:\n candidate = candidate[1:-1].strip()\n\n parsed_calls = self._parse_openrouter_text_tool_call_block(\n block=candidate,\n tools_by_name=tools_by_name,\n compat_prefix=f\"{compat_prefix}_{segment_index}\",\n )\n if parsed_calls:\n segment_index += 1\n tool_calls.extend(parsed_calls)\n continue\n\n retained_lines.append(line)\n\n stripped_text = \"\\n\".join(retained_lines).strip()\n return stripped_text, tool_calls\n\n def _parse_openrouter_text_tool_call_block(\n self,\n block: str,\n tools_by_name: dict[str, Tool],\n compat_prefix: str,\n ) -> list[dict[str, Any]]:\n \"\"\"Parse a single textual tool-call block like [tool(arg='x')].\"\"\"\n try:\n parsed = ast.parse(block.strip(), mode=\"eval\").body\n except SyntaxError:\n return []\n\n call_nodes = parsed.elts if isinstance(parsed, ast.List) else [parsed]\n tool_calls: list[dict[str, Any]] = []\n\n for call_index, call_node in enumerate(call_nodes):\n if not isinstance(call_node, ast.Call) or not isinstance(call_node.func, ast.Name):\n continue\n\n tool_name = call_node.func.id\n tool = tools_by_name.get(tool_name)\n if tool is None:\n continue\n\n try:\n tool_input = self._parse_openrouter_text_tool_call_arguments(\n call_node=call_node,\n tool=tool,\n )\n except (ValueError, SyntaxError):\n continue\n\n tool_calls.append(\n {\n \"id\": f\"{compat_prefix}_{call_index}\",\n \"name\": tool_name,\n \"input\": tool_input,\n }\n )\n\n return tool_calls\n\n @staticmethod\n def _parse_openrouter_text_tool_call_arguments(\n call_node: ast.Call,\n tool: Tool,\n ) -> dict[str, Any]:\n \"\"\"Parse positional/keyword args from a textual tool call.\"\"\"\n properties = tool.parameters.get(\"properties\", {})\n positional_keys = list(properties.keys())\n tool_input: dict[str, Any] = {}\n\n if len(call_node.args) > len(positional_keys):\n raise ValueError(\"Too many positional args for textual tool call\")\n\n for idx, arg_node in enumerate(call_node.args):\n tool_input[positional_keys[idx]] = ast.literal_eval(arg_node)\n\n for kwarg in call_node.keywords:\n if kwarg.arg is None:\n raise ValueError(\"Star args are not supported in textual tool calls\")\n tool_input[kwarg.arg] = ast.literal_eval(kwarg.value)\n\n return tool_input\n\n def _build_openrouter_tool_compat_messages(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n ) -> list[dict[str, Any]]:\n \"\"\"Build a JSON-only prompt for models without native tool support.\"\"\"\n tool_specs = [\n {\n \"name\": tool.name,\n \"description\": tool.description,\n \"parameters\": tool.parameters,\n }\n for tool in tools\n ]\n compat_instruction = (\n \"Tool compatibility mode is active because this OpenRouter model does not support \"\n \"native function calling on the routed provider.\\n\"\n \"Return exactly one JSON object and nothing else.\\n\"\n 'Schema: {\"assistant_response\": string, '\n '\"tool_calls\": [{\"name\": string, \"arguments\": object}]}\\n'\n \"Rules:\\n\"\n \"- If a tool is required, put one or more entries in tool_calls \"\n \"and do not invent tool results.\\n\"\n \"- If no tool is required, set tool_calls to [] and put the full \"\n \"answer in assistant_response.\\n\"\n \"- Only use tool names from the allowed tool list.\\n\"\n \"- arguments must always be valid JSON objects.\\n\"\n f\"Allowed tools:\\n{json.dumps(tool_specs, ensure_ascii=True)}\"\n )\n compat_system = compat_instruction if not system else f\"{system}\\n\\n{compat_instruction}\"\n\n full_messages: list[dict[str, Any]] = [{\"role\": \"system\", \"content\": compat_system}]\n full_messages.extend(messages)\n return [\n message\n for message in full_messages\n if not (\n message.get(\"role\") == \"assistant\"\n and not message.get(\"content\")\n and not message.get(\"tool_calls\")\n )\n ]\n\n async def _acomplete_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> LLMResponse:\n \"\"\"Emulate tool calling via JSON when OpenRouter rejects native tools.\"\"\"\n full_messages = self._build_openrouter_tool_compat_messages(messages, system, tools)\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n **self.extra_kwargs,\n }\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n\n response = await self._acompletion_with_rate_limit_retry(**kwargs)\n raw_content = response.choices[0].message.content or \"\"\n assistant_text, tool_calls = self._parse_openrouter_tool_compat_response(\n raw_content,\n tools,\n )\n usage = response.usage\n input_tokens = usage.prompt_tokens if usage else 0\n output_tokens = usage.completion_tokens if usage else 0\n stop_reason = \"tool_calls\" if tool_calls else (response.choices[0].finish_reason or \"stop\")\n\n return LLMResponse(\n content=assistant_text,\n model=response.model or self.model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\n \"compat_mode\": \"openrouter_tool_emulation\",\n \"tool_calls\": tool_calls,\n \"response\": response,\n },\n )\n\n async def _stream_via_openrouter_tool_compat(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool],\n max_tokens: int,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback stream for OpenRouter models without native tool support.\"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n logger.info(\n \"[openrouter-tool-compat] Using compatibility mode for %s\",\n self.model,\n )\n try:\n response = await self._acomplete_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw_response = response.raw_response if isinstance(response.raw_response, dict) else {}\n tool_calls = raw_response.get(\"tool_calls\", [])\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n for tool_call in tool_calls:\n yield ToolCallEvent(\n tool_use_id=tool_call[\"id\"],\n tool_name=tool_call[\"name\"],\n tool_input=tool_call[\"input\"],\n )\n\n yield FinishEvent(\n stop_reason=response.stop_reason,\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def _stream_via_nonstream_completion(\n self,\n messages: list[dict[str, Any]],\n system: str,\n tools: list[Tool] | None,\n max_tokens: int,\n response_format: dict[str, Any] | None,\n json_mode: bool,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Fallback path: convert non-stream completion to stream events.\n\n Some providers currently fail in LiteLLM's chunk parser for stream=True.\n For those providers we do a regular async completion and emit equivalent\n stream events so higher layers continue to work.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n try:\n response = await self.acomplete(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n )\n except Exception as e:\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n raw = response.raw_response\n tool_calls = []\n if raw and hasattr(raw, \"choices\") and raw.choices:\n msg = raw.choices[0].message\n tool_calls = msg.tool_calls or []\n\n for tc in tool_calls:\n args = tc.function.arguments if tc.function else \"\"\n parsed_args = self._parse_tool_call_arguments(\n args,\n tc.function.name if tc.function else \"\",\n )\n yield ToolCallEvent(\n tool_use_id=getattr(tc, \"id\", \"\"),\n tool_name=tc.function.name if tc.function else \"\",\n tool_input=parsed_args,\n )\n\n if response.content:\n yield TextDeltaEvent(content=response.content, snapshot=response.content)\n yield TextEndEvent(full_text=response.content)\n\n yield FinishEvent(\n stop_reason=response.stop_reason or \"stop\",\n input_tokens=response.input_tokens,\n output_tokens=response.output_tokens,\n model=response.model,\n )\n\n async def stream(\n self,\n messages: list[dict[str, Any]],\n system: str = \"\",\n tools: list[Tool] | None = None,\n max_tokens: int = 4096,\n response_format: dict[str, Any] | None = None,\n json_mode: bool = False,\n ) -> AsyncIterator[StreamEvent]:\n \"\"\"Stream a completion via litellm.acompletion(stream=True).\n\n Yields StreamEvent objects as chunks arrive from the provider.\n Tool call arguments are accumulated across chunks and yielded as\n a single ToolCallEvent with fully parsed JSON when complete.\n\n Empty responses (e.g. Gemini stealth rate-limits that return 200\n with no content) are retried with exponential backoff, mirroring\n the retry behaviour of ``_completion_with_rate_limit_retry``.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n TextEndEvent,\n ToolCallEvent,\n )\n\n # MiniMax currently fails in litellm's stream chunk parser for some\n # responses (missing \"id\" in stream chunks). Use non-stream fallback.\n if self._is_minimax_model():\n async for event in self._stream_via_nonstream_completion(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n response_format=response_format,\n json_mode=json_mode,\n ):\n yield event\n return\n\n if tools and self._is_openrouter_model() and _is_openrouter_tool_compat_cached(self.model):\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools,\n max_tokens=max_tokens,\n ):\n yield event\n return\n\n full_messages: list[dict[str, Any]] = []\n if self._claude_code_oauth:\n billing = _claude_code_billing_header(messages)\n full_messages.append({\"role\": \"system\", \"content\": billing})\n if system:\n sys_msg: dict[str, Any] = {\"role\": \"system\", \"content\": system}\n if _model_supports_cache_control(self.model):\n sys_msg[\"cache_control\"] = {\"type\": \"ephemeral\"}\n full_messages.append(sys_msg)\n full_messages.extend(messages)\n\n # Codex Responses API requires an `instructions` field (system prompt).\n # Inject a minimal one when callers don't provide a system message.\n if self._codex_backend and not any(m[\"role\"] == \"system\" for m in full_messages):\n full_messages.insert(0, {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"})\n\n # Add JSON mode via prompt engineering (works across all providers)\n if json_mode:\n json_instruction = \"\\n\\nPlease respond with a valid JSON object.\"\n if full_messages and full_messages[0][\"role\"] == \"system\":\n full_messages[0][\"content\"] += json_instruction\n else:\n full_messages.insert(0, {\"role\": \"system\", \"content\": json_instruction.strip()})\n\n # Remove ghost empty assistant messages (content=\"\" and no tool_calls).\n # These arise when a model returns an empty stream after a tool result\n # (an \"expected\" no-op turn). Keeping them in history confuses some\n # models (notably Codex/gpt-5.3) and causes cascading empty streams.\n full_messages = [\n m\n for m in full_messages\n if not (\n m.get(\"role\") == \"assistant\" and not m.get(\"content\") and not m.get(\"tool_calls\")\n )\n ]\n\n kwargs: dict[str, Any] = {\n \"model\": self.model,\n \"messages\": full_messages,\n \"max_tokens\": max_tokens,\n \"stream\": True,\n **self.extra_kwargs,\n }\n # stream_options is OpenAI-specific; Anthropic rejects it with 400.\n # Only include it for providers that support it.\n if not self._is_anthropic_model():\n kwargs[\"stream_options\"] = {\"include_usage\": True}\n if self.api_key:\n kwargs[\"api_key\"] = self.api_key\n if self.api_base:\n kwargs[\"api_base\"] = self.api_base\n if tools:\n kwargs[\"tools\"] = [self._tool_to_openai_format(t) for t in tools]\n if _is_ollama_model(self.model):\n # Ollama requires explicit tool_choice=auto for function calling\n # so future readers don't have to guess.\n kwargs.setdefault(\"tool_choice\", \"auto\")\n if response_format:\n kwargs[\"response_format\"] = response_format\n # The Codex ChatGPT backend (Responses API) rejects several params.\n if self._codex_backend:\n kwargs.pop(\"max_tokens\", None)\n kwargs.pop(\"stream_options\", None)\n\n for attempt in range(RATE_LIMIT_MAX_RETRIES + 1):\n # Post-stream events (ToolCall, TextEnd, Finish) are buffered\n # because they depend on the full stream. TextDeltaEvents are\n # yielded immediately so callers see tokens in real time.\n tail_events: list[StreamEvent] = []\n accumulated_text = \"\"\n tool_calls_acc: dict[int, dict[str, str]] = {}\n _last_tool_idx = 0 # tracks most recently opened tool call slot\n input_tokens = 0\n output_tokens = 0\n stream_finish_reason: str | None = None\n\n try:\n response = await litellm.acompletion(**kwargs) # type: ignore[union-attr]\n\n async for chunk in response:\n # Capture usage from the trailing usage-only chunk that\n # stream_options={\"include_usage\": True} sends with empty choices.\n if not chunk.choices:\n usage = getattr(chunk, \"usage\", None)\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n logger.debug(\n \"[tokens] trailing usage chunk: input=%d output=%d model=%s\",\n input_tokens,\n output_tokens,\n self.model,\n )\n else:\n logger.debug(\n \"[tokens] empty-choices chunk with no usage (model=%s)\",\n self.model,\n )\n continue\n choice = chunk.choices[0]\n\n delta = choice.delta\n\n # --- Text content — yield immediately for real-time streaming ---\n if delta and delta.content:\n accumulated_text += delta.content\n yield TextDeltaEvent(\n content=delta.content,\n snapshot=accumulated_text,\n )\n\n # --- Tool calls (accumulate across chunks) ---\n # The Codex/Responses API bridge (litellm bug) hardcodes\n # index=0 on every ChatCompletionToolCallChunk, even for\n # parallel tool calls. We work around this by using tc.id\n # (set on output_item.added events) as a \"new tool call\"\n # signal and tracking the most recently opened slot for\n # argument deltas that arrive with id=None.\n if delta and delta.tool_calls:\n for tc in delta.tool_calls:\n idx = tc.index if hasattr(tc, \"index\") and tc.index is not None else 0\n\n if tc.id:\n # New tool call announced (or done event re-sent).\n # Check if this id already has a slot.\n existing_idx = next(\n (k for k, v in tool_calls_acc.items() if v[\"id\"] == tc.id),\n None,\n )\n if existing_idx is not None:\n idx = existing_idx\n elif idx in tool_calls_acc and tool_calls_acc[idx][\"id\"] not in (\n \"\",\n tc.id,\n ):\n # Slot taken by a different call — assign new index\n idx = max(tool_calls_acc.keys()) + 1\n _last_tool_idx = idx\n else:\n # Argument delta with no id — route to last opened slot\n idx = _last_tool_idx\n\n if idx not in tool_calls_acc:\n tool_calls_acc[idx] = {\"id\": \"\", \"name\": \"\", \"arguments\": \"\"}\n if tc.id:\n tool_calls_acc[idx][\"id\"] = tc.id\n if tc.function:\n if tc.function.name:\n tool_calls_acc[idx][\"name\"] = tc.function.name\n if tc.function.arguments:\n tool_calls_acc[idx][\"arguments\"] += tc.function.arguments\n\n # --- Finish ---\n if choice.finish_reason:\n stream_finish_reason = choice.finish_reason\n for _idx, tc_data in sorted(tool_calls_acc.items()):\n parsed_args = self._parse_tool_call_arguments(\n tc_data.get(\"arguments\", \"\"),\n tc_data.get(\"name\", \"\"),\n )\n tail_events.append(\n ToolCallEvent(\n tool_use_id=tc_data[\"id\"],\n tool_name=tc_data[\"name\"],\n tool_input=parsed_args,\n )\n )\n\n if accumulated_text:\n tail_events.append(TextEndEvent(full_text=accumulated_text))\n\n usage = getattr(chunk, \"usage\", None)\n logger.debug(\n \"[tokens] finish-chunk raw usage: %r (type=%s)\",\n usage,\n type(usage).__name__,\n )\n cached_tokens = 0\n if usage:\n input_tokens = getattr(usage, \"prompt_tokens\", 0) or 0\n output_tokens = getattr(usage, \"completion_tokens\", 0) or 0\n _details = getattr(usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] finish-chunk usage: \"\n \"input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n\n logger.debug(\n \"[tokens] finish event: input=%d output=%d cached=%d stop=%s model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n choice.finish_reason,\n self.model,\n )\n tail_events.append(\n FinishEvent(\n stop_reason=choice.finish_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=self.model,\n )\n )\n\n # Fallback: LiteLLM strips usage from yielded chunks before\n # returning them to us, but appends the original chunk (with\n # usage intact) to response.chunks first. Use LiteLLM's own\n # calculate_total_usage() on that accumulated list.\n if input_tokens == 0 and output_tokens == 0:\n try:\n from litellm.litellm_core_utils.streaming_handler import (\n calculate_total_usage,\n )\n\n _chunks = getattr(response, \"chunks\", None)\n if _chunks:\n _usage = calculate_total_usage(chunks=_chunks)\n input_tokens = _usage.prompt_tokens or 0\n output_tokens = _usage.completion_tokens or 0\n _details = getattr(_usage, \"prompt_tokens_details\", None)\n cached_tokens = (\n getattr(_details, \"cached_tokens\", 0) or 0\n if _details is not None\n else getattr(_usage, \"cache_read_input_tokens\", 0) or 0\n )\n logger.debug(\n \"[tokens] post-loop chunks fallback:\"\n \" input=%d output=%d cached=%d model=%s\",\n input_tokens,\n output_tokens,\n cached_tokens,\n self.model,\n )\n # Patch the FinishEvent already queued with 0 tokens\n for _i, _ev in enumerate(tail_events):\n if isinstance(_ev, FinishEvent) and _ev.input_tokens == 0:\n tail_events[_i] = FinishEvent(\n stop_reason=_ev.stop_reason,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n cached_tokens=cached_tokens,\n model=_ev.model,\n )\n break\n except Exception as _e:\n logger.debug(\"[tokens] chunks fallback failed: %s\", _e)\n\n # Check whether the stream produced any real content.\n # (If text deltas were yielded above, has_content is True\n # and we skip the retry path — nothing was yielded in vain.)\n has_content = accumulated_text or tool_calls_acc\n if not has_content:\n # finish_reason=length means the model exhausted\n # max_tokens before producing content. Retrying with\n # the same max_tokens will never help.\n if stream_finish_reason == \"length\":\n max_tok = kwargs.get(\"max_tokens\", \"unset\")\n logger.error(\n f\"[stream] {self.model} returned empty content \"\n f\"with finish_reason=length \"\n f\"(max_tokens={max_tok}). The model exhausted \"\n f\"its token budget before producing visible \"\n f\"output. Increase max_tokens or use a \"\n f\"different model. Not retrying.\"\n )\n for event in tail_events:\n yield event\n return\n\n # Empty stream — always retry regardless of last message\n # role. Ghost empty streams after tool results are NOT\n # expected no-ops; they create infinite loops when the\n # conversation doesn't change between iterations.\n # After retries, return the empty result and let the\n # caller (EventLoopNode) decide how to handle it.\n last_role = next(\n (m[\"role\"] for m in reversed(full_messages) if m.get(\"role\") != \"system\"),\n None,\n )\n if attempt < EMPTY_STREAM_MAX_RETRIES:\n token_count, token_method = _estimate_tokens(\n self.model,\n full_messages,\n )\n dump_path = _dump_failed_request(\n model=self.model,\n kwargs=kwargs,\n error_type=\"empty_stream\",\n attempt=attempt,\n )\n logger.warning(\n f\"[stream-retry] {self.model} returned empty stream \"\n f\"after {last_role} message — \"\n f\"~{token_count} tokens ({token_method}). \"\n f\"Request dumped to: {dump_path}. \"\n f\"Retrying in {EMPTY_STREAM_RETRY_DELAY}s \"\n f\"(attempt {attempt + 1}/{EMPTY_STREAM_MAX_RETRIES})\"\n )\n await asyncio.sleep(EMPTY_STREAM_RETRY_DELAY)\n continue\n\n # All retries exhausted — log and return the empty\n # result. EventLoopNode's empty response guard will\n # accept if all outputs are set, or handle the ghost\n # stream case if outputs are still missing.\n logger.error(\n f\"[stream] {self.model} returned empty stream after \"\n f\"{EMPTY_STREAM_MAX_RETRIES} retries \"\n f\"(last_role={last_role}). Returning empty result.\"\n )\n\n # Success (or empty after exhausted retries) — flush events.\n for event in tail_events:\n yield event\n return\n\n except RateLimitError as e:\n if attempt < RATE_LIMIT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} rate limited (429): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{RATE_LIMIT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n yield StreamErrorEvent(error=str(e), recoverable=False)\n return\n\n except Exception as e:\n if self._should_use_openrouter_tool_compat(e, tools):\n _remember_openrouter_tool_compat_model(self.model)\n async for event in self._stream_via_openrouter_tool_compat(\n messages=messages,\n system=system,\n tools=tools or [],\n max_tokens=max_tokens,\n ):\n yield event\n return\n if _is_stream_transient_error(e) and attempt < STREAM_TRANSIENT_MAX_RETRIES:\n wait = _compute_retry_delay(attempt, exception=e)\n logger.warning(\n f\"[stream-retry] {self.model} transient error \"\n f\"({type(e).__name__}): {e!s}. \"\n f\"Retrying in {wait:.1f}s \"\n f\"(attempt {attempt + 1}/{STREAM_TRANSIENT_MAX_RETRIES})\"\n )\n await asyncio.sleep(wait)\n continue\n recoverable = _is_stream_transient_error(e)\n yield StreamErrorEvent(error=str(e), recoverable=recoverable)\n return\n\n async def _collect_stream_to_response(\n self,\n stream: AsyncIterator[StreamEvent],\n ) -> LLMResponse:\n \"\"\"Consume a stream() iterator and collect it into a single LLMResponse.\n\n Used by acomplete() to route through the unified streaming path so that\n all backends (including Codex) get proper tool call handling.\n \"\"\"\n from framework.llm.stream_events import (\n FinishEvent,\n StreamErrorEvent,\n TextDeltaEvent,\n ToolCallEvent,\n )\n\n content = \"\"\n tool_calls: list[dict[str, Any]] = []\n input_tokens = 0\n output_tokens = 0\n stop_reason = \"\"\n model = self.model\n\n async for event in stream:\n if isinstance(event, TextDeltaEvent):\n content = event.snapshot # snapshot is the accumulated text\n elif isinstance(event, ToolCallEvent):\n tool_calls.append(\n {\n \"id\": event.tool_use_id,\n \"name\": event.tool_name,\n \"input\": event.tool_input,\n }\n )\n elif isinstance(event, FinishEvent):\n input_tokens = event.input_tokens\n output_tokens = event.output_tokens\n stop_reason = event.stop_reason\n if event.model:\n model = event.model\n elif isinstance(event, StreamErrorEvent):\n if not event.recoverable:\n raise RuntimeError(f\"Stream error: {event.error}\")\n\n return LLMResponse(\n content=content,\n model=model,\n input_tokens=input_tokens,\n output_tokens=output_tokens,\n stop_reason=stop_reason,\n raw_response={\"tool_calls\": tool_calls} if tool_calls else None,\n )\n"}}}} |