1 line
50 KiB
JSON
1 line
50 KiB
JSON
{"repo": "bernardolab/medevalarena", "files": {"quizbench/clients.py": {"d2a7ba1fee997129ed3d5b37355fbf849395ce41": {"t": 1785745440.744056, "n": 24453, "h": "9a16ab617cfaccfb", "c": "import json\nimport logging\nimport os\nimport requests\n\nfrom typing import Optional\n\nfrom tenacity import (\n retry,\n retry_if_exception_type,\n stop_after_attempt,\n wait_exponential,\n before_sleep_log,\n)\n\n# OpenAI\nfrom openai import OpenAI as _OpenAI\n# Anthropic\nimport anthropic\nfrom anthropic import transform_schema\n\n# Google\nfrom quizbench.utils_grok import call_grok_sync, is_grok_model\nfrom pydantic import BaseModel, ValidationError, Field\nfrom typing import Literal\nimport sys\n\nlogger = logging.getLogger(\"quizbench.clients\")\n\n\n# --- Helpers ---\nclass JudgeExtraction(BaseModel):\n analysis: str = Field(\n description=\"Detailed analysis of the medical reasoning and logical validity.\"\n )\n medical_accuracy_score: Literal[\"1\", \"2\", \"3\", \"4\", \"5\"] = Field(\n description=\"Integer (1–5) indicating medical accuracy.\"\n )\n logical_validity: bool = Field(\n description=\"True if the reasoning is logically sound.\"\n )\n logical_false_reason: Literal[\"C\", \"N\", \"M\", \"U\", \"K\", \"T\"] = Field(\n description=(\n \"One-letter code for logical status: \"\n \"C=Contradiction, N=No answer defensible, \"\n \"M=Multiple answers defensible, U=Underspecified, \"\n \"K=Miskeyed, T=Used only when logical_validity is true.\"\n )\n )\n\nfrom typing import Optional\nimport json\n\n\ndef _extract_text_from_responses_output_grok(resp) -> Optional[str]:\n \"\"\"\n Best-effort extractor for OpenAI Responses API objects and ChatCompletion objects.\n Walks output/choices and returns the first text payload found.\n \"\"\"\n # 1. Native Responses API: resp.output\n output = getattr(resp, \"output\", None)\n\n # 2. Fallback: dict-style \"output\"\n if output is None and isinstance(resp, dict):\n output = resp.get(\"output\")\n\n # 3. Chat Completions API: resp.choices[*].message.content\n # We normalize choices into the same \"output\" shape this helper expects:\n # a list of items each with a \"content\" field.\n if output is None and hasattr(resp, \"choices\"):\n output = []\n for choice in getattr(resp, \"choices\") or []:\n # choice can be an object or dict\n message = getattr(choice, \"message\", None)\n if message is None and isinstance(choice, dict):\n message = choice.get(\"message\")\n if message is None:\n continue\n\n content = getattr(message, \"content\", None)\n if content is None and isinstance(message, dict):\n content = message.get(\"content\")\n\n if content is not None:\n output.append({\"content\": content})\n\n # 4. Still nothing? Bail out.\n if not output:\n return None\n\n # 5. Existing extraction logic\n for item in output:\n content = item.get(\"content\") if isinstance(item, dict) else getattr(item, \"content\", None)\n if not content:\n continue\n\n # Simple string content\n if isinstance(content, str):\n cleaned = content.strip()\n if cleaned:\n return cleaned\n continue\n\n # List of structured content parts (Responses API or new ChatCompletion v1)\n if isinstance(content, list):\n for part in content:\n # Prefer explicit text fields\n text = part.get(\"text\") if isinstance(part, dict) else getattr(part, \"text\", None)\n if text:\n return text\n\n # Fall back to parsed objects if present\n parsed = part.get(\"parsed\") if isinstance(part, dict) else getattr(part, \"parsed\", None)\n if parsed is not None:\n if hasattr(parsed, \"model_dump_json\"):\n return parsed.model_dump_json()\n if hasattr(parsed, \"dict\"):\n return json.dumps(parsed.dict())\n try:\n return json.dumps(parsed)\n except Exception as exc: # noqa: BLE001\n # Hard-fail rather than silently skipping ambiguous parsed content\n raise RuntimeError(\n f\"Failed to serialize parsed Responses output: {parsed!r}\"\n ) from exc\n\n # Handle unexpected content containers that still expose text/value\n maybe_text = getattr(content, \"text\", None) or getattr(content, \"value\", None)\n if maybe_text:\n return str(maybe_text)\n\n return None\n\n\ndef _extract_text_from_openrouter_kimi(resp_json: dict) -> Optional[str]:\n \"\"\"\n Extract text content from an OpenRouter /chat/completions JSON response\n for Kimi (moonshotai) models, where the payload is a plain dict with\n choices[*].message.content.\n \"\"\"\n if not isinstance(resp_json, dict):\n return None\n\n choices = resp_json.get(\"choices\") or []\n if not choices:\n return None\n\n for choice in choices:\n if not isinstance(choice, dict):\n continue\n message = choice.get(\"message\") or {}\n if not isinstance(message, dict):\n continue\n\n content = message.get(\"content\")\n if not content:\n continue\n\n # Kimi via OpenRouter commonly returns a single JSON string here.\n if isinstance(content, str):\n cleaned = content.strip()\n if cleaned:\n return cleaned\n continue\n\n # Fallback: list of structured parts, e.g. [{\"type\": \"text\", \"text\": \"...\"}]\n if isinstance(content, list):\n parts: list[str] = []\n for part in content:\n if isinstance(part, dict):\n text = part.get(\"text\")\n if isinstance(text, str):\n parts.append(text)\n else:\n inner_content = part.get(\"content\")\n if isinstance(inner_content, str):\n parts.append(inner_content)\n else:\n parts.append(str(part))\n elif isinstance(part, str):\n parts.append(part)\n joined = \"\".join(parts).strip()\n if joined:\n return joined\n\n return None\n\n\n\ndef _extract_text_from_responses_output(resp) -> Optional[str]:\n \"\"\"\n Best-effort extractor for OpenAI Responses API objects.\n Walks output items and returns the first text payload found.\n \"\"\"\n output = getattr(resp, \"output\", None)\n if not output:\n return None\n\n for item in output:\n content = item.get(\"content\") if isinstance(item, dict) else getattr(item, \"content\", None)\n if not content:\n continue\n\n if isinstance(content, str):\n cleaned = content.strip()\n if cleaned:\n return cleaned\n continue\n\n if isinstance(content, list):\n for part in content:\n # Prefer explicit text fields\n text = part.get(\"text\") if isinstance(part, dict) else getattr(part, \"text\", None)\n if text:\n return text\n\n # Fall back to parsed objects if present\n parsed = part.get(\"parsed\") if isinstance(part, dict) else getattr(part, \"parsed\", None)\n if parsed is not None:\n if hasattr(parsed, \"model_dump_json\"):\n return parsed.model_dump_json()\n if hasattr(parsed, \"dict\"):\n return json.dumps(parsed.dict())\n try:\n return json.dumps(parsed)\n except Exception as exc: # noqa: BLE001\n # Hard-fail rather than silently skipping ambiguous parsed content\n raise RuntimeError(\n f\"Failed to serialize parsed Responses output: {parsed!r}\"\n ) from exc\n\n # Handle unexpected content containers that still expose text/value\n maybe_text = getattr(content, \"text\", None) or getattr(content, \"value\", None)\n if maybe_text:\n return str(maybe_text)\n\n return None\n\ndef _raise_if_missing(var: str) -> None:\n if not os.getenv(var):\n raise RuntimeError(f\"Missing environment variable: {var}\")\n\n\ndef _build_openrouter_input_from_prompt(prompt: str) -> list[dict]:\n \"\"\"\n Build a minimal OpenRouter `input` payload for a single\n text-only user prompt, matching the Responses API schema.\n \"\"\"\n return [\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": prompt,\n }\n ],\n }\n ]\n\n\n# --- LLM call dispatcher ---\n@retry(\n reraise=True,\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n retry=retry_if_exception_type(Exception),\n before_sleep=before_sleep_log(logger, logging.WARNING),\n)\ndef call_llm(\n model_name: str,\n prompt: str,\n max_tokens: int = 1200,\n temperature: float = 0.0,\n openrouter: bool = False,\n judge_mode: bool = False,\n):\n \"\"\"\n Unified text-only call.\n\n Returns a tuple (text, reasoning_trace) where:\n - text is the primary string content used by callers.\n - reasoning_trace is a provider-specific raw payload when available\n (DeepSeek/Kimi via OpenRouter), otherwise None.\n \"\"\"\n mn = model_name.strip()\n\n if is_grok_model(mn):\n logger.info(\n \"Calling Grok model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n\n if judge_mode:\n # Using async single call (not pseudo-batched)\n client = _OpenAI(api_key=os.getenv(\"XAI_API_KEY\"), base_url=\"https://api.x.ai/v1\")\n resp = client.chat.completions.create(\n model=mn,\n max_tokens=max_tokens,\n temperature=temperature,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n )\n text = _extract_text_from_responses_output_grok(resp)\n if text:\n return text, None\n raise RuntimeError(f\"Responses API returned no usable text content: {resp!r}\")\n\n text = call_grok_sync(\n model_name=mn,\n prompt=prompt,\n max_tokens=max_tokens,\n temperature=temperature,\n )\n return text, None\n\n # OpenAI gpt-5 family (Responses API)\n if mn.startswith(\"gpt-5\"):\n _raise_if_missing(\"OPENAI_API_KEY\")\n logger.info(\n \"Calling OpenAI Responses API for model '%s' (max_tokens=%d, temperature=1.000).\",\n mn,\n max_tokens,\n )\n client = _OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n\n if judge_mode:\n resp = client.responses.parse(\n model=mn,\n input=[{\"role\": \"user\", \"content\": prompt}],\n temperature=1, # Responses API requires temperature=1 for gpt-5 family\n max_output_tokens=max_tokens,\n top_p=1,\n text_format=JudgeExtraction,\n )\n\n text = _extract_text_from_responses_output(resp)\n if text:\n return text, None\n\n # As a last resort, stringify the whole payload so the caller can attempt to parse\n return str(resp), None\n \n # Eval mode\n resp = client.responses.create(**req)\n if resp.output_text:\n return resp.output_text, None\n\n # If we get here, treat it as a hard failure\n raise RuntimeError(f\"Responses API returned no usable text content: {resp!r}\")\n\n # OpenAI chat models (gpt-4/4o/etc.)\n if mn.startswith(\"gpt-\"):\n _raise_if_missing(\"OPENAI_API_KEY\")\n logger.info(\n \"Calling OpenAI model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n client = _OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n if model_name == \"gpt-4o\":\n resp = client.chat.completions.create(\n model=mn,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=temperature,\n max_tokens=max_tokens,\n top_p=1,\n )\n else:\n resp = client.chat.completions.create(\n model=mn,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=1, # gpt-5 family only supports temperature 1\n max_completion_tokens=max_tokens,\n top_p=1,\n )\n print(repr(resp))\n return resp.choices[0].message.content, None\n\n # Anthropic\n if mn.startswith(\"claude-3\") or mn.startswith(\"claude-opus\"):\n _raise_if_missing(\"ANTHROPIC_API_KEY\")\n logger.info(\n \"Calling Anthropic model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n a = anthropic.Anthropic(api_key=os.getenv(\"ANTHROPIC_API_KEY\"))\n\n if judge_mode:\n from pydantic import BaseModel # type: ignore[redefined-outer-name]\n from anthropic import transform_schema # type: ignore[unused-import]\n\n response = a.beta.messages.create(\n model=\"claude-sonnet-4-5\", #mn,\n betas=[\"structured-outputs-2025-11-13\"],\n max_tokens=max_tokens,\n temperature=temperature,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n output_format={\n \"type\": \"json_schema\",\n \"schema\": transform_schema(JudgeExtraction),\n }\n )\n\n # parsed = JudgeExtraction.model_validate_json(response.text)\n return response.content[0].text, None\n\n msg = a.messages.create(\n model=mn,\n max_tokens=max_tokens,\n temperature=temperature,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n )\n return msg.content[0].text, None\n\n # Google Gemini\n if mn.startswith(\"gemini-\"):\n from google import genai \n from google.genai import types\n\n _raise_if_missing(\"GOOGLE_API_KEY\")\n logger.info(\n \"Calling Gemini model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n\n if judge_mode:\n try:\n client = genai.Client(api_key=os.getenv(\"GOOGLE_API_KEY\")) # type: ignore[name-defined]\n response = client.models.generate_content(\n model=mn,\n contents=prompt,\n config={\n \"temperature\": temperature,\n \"top_p\": 1,\n \"max_output_tokens\": max_tokens,\n \"response_mime_type\": \"application/json\",\n \"response_schema\": JudgeExtraction.model_json_schema(),\n },\n )\n\n parsed = JudgeExtraction.model_validate_json(response.text)\n return parsed.model_dump_json(), None\n\n except Exception as e: # noqa: BLE001\n logger.error(f\"Gemini API call failed: {e}\")\n try:\n if hasattr(response, \"prompt_feedback\"):\n logger.error(f\"Feedback: {response.prompt_feedback}\")\n if hasattr(response, \"candidates\"):\n logger.error(f\"Finish Reason: {response.candidates[0].finish_reason}\")\n except Exception: # noqa: BLE001\n pass\n raise\n\n client = genai.Client(api_key=os.getenv(\"GOOGLE_API_KEY\"))\n response = client.models.generate_content(\n model=mn,\n contents=prompt,\n config={\n \"temperature\": temperature,\n \"top_p\": 1,\n \"max_output_tokens\": max_tokens,\n },\n safety_settings=None,\n )\n return response.text, None\n\n # DeepSeek or Kimi (optionally via OpenRouter)\n mn_lower = mn.lower()\n if mn_lower.startswith(\"deepseek-\") or mn_lower.startswith(\"kimi-\"):\n print(f\" [INFO]: client.py detected {mn_lower}\")\n\n mn_fam = mn_lower.split(\"-\")[0]\n\n if mn_fam == \"kimi\":\n mn_fam = \"moonshotai\"\n\n\n if openrouter:\n print(\" [INFO]: Using Openrouter\")\n _raise_if_missing(\"OPENROUTER_API_KEY\")\n logger.info(\n \"Calling %s model via OpenRouter '%s' (max_tokens=%d, temperature=%.3f).\",\n mn_fam,\n mn,\n max_tokens,\n temperature,\n )\n\n api_key = os.getenv(\"OPENROUTER_API_KEY\")\n\n # Judge mode: request structured JSON with potential reasoning traces.\n if judge_mode:\n print(\"[Info] Using Openrouter with Judge mode\")\n\n # if mn_lower.startswith(\"kimi-\") or mn_lower.startswith(\"deepseek-\"):\n response = requests.post(\n \"https://openrouter.ai/api/v1/chat/completions\",\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n json={\n \"model\": f\"{mn_fam}/{mn}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": prompt,\n }\n ],\n \"response_format\": {\n \"type\": \"json_schema\", \n \"json_schema\": {\n \"name\": \"judge_extraction\",\n \"schema\": JudgeExtraction.schema(),\n \"strict\": True,\n }\n },\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n \"top_p\": 1,\n },\n )\n\n resp_json = response.json()\n\n print(\"[INFO] Debug trace:\")\n print(resp_json)\n\n text = _extract_text_from_openrouter_kimi(resp_json)\n\n\n if text:\n print(text)\n return text, resp_json\n\n print(\"[JUDGE] Error extracting response.\")\n print(\"[INFO] Debug trace:\")\n print(json.dumps(resp_json, indent=4))\n raise RuntimeError(\"Failed to extract text from OpenRouter DeepSeek/Kimi response.\")\n\n print(f\"[INFO] Using model {mn} via OpenRouter\")\n # Non-judge mode over OpenRouter: plain chat completion.\n response = requests.post(\n \"https://openrouter.ai/api/v1/chat/completions\",\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n json={\n \"model\": f\"{mn_fam}/{mn}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": prompt,\n }\n ],\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n \"top_p\": 1,\n },\n )\n\n if response.status_code != 200:\n raise RuntimeError(\n f\"OpenRouter {mn_lower} call failed \"\n f\"(status={response.status_code}): {response.text[:1000]}\"\n )\n\n result = response.json()\n\n try:\n choice = result[\"choices\"][0]\n message = choice.get(\"message\") or {}\n content = message.get(\"content\", \"\")\n if isinstance(content, list):\n parts = []\n for part in content:\n if isinstance(part, dict) and part.get(\"type\") == \"text\":\n parts.append(part.get(\"text\", \"\"))\n else:\n parts.append(str(part))\n content = \"\".join(parts)\n if not isinstance(content, str):\n content = str(content)\n except Exception as exc: # noqa: BLE001\n raise RuntimeError(\n f\"Unexpected OpenRouter response format: {result}\"\n ) from exc\n\n return content, result\n\n # DeepSeek via native API\n if mn_lower.startswith(\"deepseek-\"):\n\n\n\n\n if openrouter:\n print(f\"[INFO] Using model {mn} via OpenRouter\")\n # Non-judge mode over OpenRouter: plain chat completion.\n response = requests.post(\n \"https://openrouter.ai/api/v1/chat/completions\",\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n json={\n \"model\": f\"{mn_fam}/{mn}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": prompt,\n }\n ],\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n \"top_p\": 1,\n },\n )\n\n if response.status_code != 200:\n raise RuntimeError(\n f\"OpenRouter {mn_lower} call failed \"\n f\"(status={response.status_code}): {response.text[:1000]}\"\n )\n\n result = response.json()\n\n try:\n choice = result[\"choices\"][0]\n message = choice.get(\"message\") or {}\n content = message.get(\"content\", \"\")\n if isinstance(content, list):\n parts = []\n for part in content:\n if isinstance(part, dict) and part.get(\"type\") == \"text\":\n parts.append(part.get(\"text\", \"\"))\n else:\n parts.append(str(part))\n content = \"\".join(parts)\n if not isinstance(content, str):\n content = str(content)\n except Exception as exc: # noqa: BLE001\n raise RuntimeError(\n f\"Unexpected OpenRouter response format: {result}\"\n ) from exc\n\n return content, result\n\n\n # Legacy Deepseek API\n _raise_if_missing(\"DEEPSEEK_API_KEY\")\n logger.info(\n \"Calling DeepSeek model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n deepseek_model_name = \"deepseek-reasoner\"\n\n api_key = os.getenv(\"DEEPSEEK_API_KEY\")\n\n client = _OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com/v1\", timeout=10)\n\n resp = client.chat.completions.create(\n model=deepseek_model_name,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=temperature,\n max_tokens=max_tokens,\n top_p=1,\n )\n\n print(resp)\n return resp.choices[0].message.content, None\n\n # Placeholder other models\n raise NotImplementedError(\n f\"Model '{mn}' is not wired correctly. Add provider support in quizbench/clients.py.\"\n ) \n\n # Placeholder for other providers\n raise NotImplementedError(\n f\"Model '{mn}' is not wired. Add provider support in quizbench/clients.py.\"\n )\n"}, "efc71b60e99811289c088855c0ccec110f7a2b48": {"t": 1785753246.896802, "n": 24453, "h": "9a16ab617cfaccfb", "c": "import json\nimport logging\nimport os\nimport requests\n\nfrom typing import Optional\n\nfrom tenacity import (\n retry,\n retry_if_exception_type,\n stop_after_attempt,\n wait_exponential,\n before_sleep_log,\n)\n\n# OpenAI\nfrom openai import OpenAI as _OpenAI\n# Anthropic\nimport anthropic\nfrom anthropic import transform_schema\n\n# Google\nfrom quizbench.utils_grok import call_grok_sync, is_grok_model\nfrom pydantic import BaseModel, ValidationError, Field\nfrom typing import Literal\nimport sys\n\nlogger = logging.getLogger(\"quizbench.clients\")\n\n\n# --- Helpers ---\nclass JudgeExtraction(BaseModel):\n analysis: str = Field(\n description=\"Detailed analysis of the medical reasoning and logical validity.\"\n )\n medical_accuracy_score: Literal[\"1\", \"2\", \"3\", \"4\", \"5\"] = Field(\n description=\"Integer (1–5) indicating medical accuracy.\"\n )\n logical_validity: bool = Field(\n description=\"True if the reasoning is logically sound.\"\n )\n logical_false_reason: Literal[\"C\", \"N\", \"M\", \"U\", \"K\", \"T\"] = Field(\n description=(\n \"One-letter code for logical status: \"\n \"C=Contradiction, N=No answer defensible, \"\n \"M=Multiple answers defensible, U=Underspecified, \"\n \"K=Miskeyed, T=Used only when logical_validity is true.\"\n )\n )\n\nfrom typing import Optional\nimport json\n\n\ndef _extract_text_from_responses_output_grok(resp) -> Optional[str]:\n \"\"\"\n Best-effort extractor for OpenAI Responses API objects and ChatCompletion objects.\n Walks output/choices and returns the first text payload found.\n \"\"\"\n # 1. Native Responses API: resp.output\n output = getattr(resp, \"output\", None)\n\n # 2. Fallback: dict-style \"output\"\n if output is None and isinstance(resp, dict):\n output = resp.get(\"output\")\n\n # 3. Chat Completions API: resp.choices[*].message.content\n # We normalize choices into the same \"output\" shape this helper expects:\n # a list of items each with a \"content\" field.\n if output is None and hasattr(resp, \"choices\"):\n output = []\n for choice in getattr(resp, \"choices\") or []:\n # choice can be an object or dict\n message = getattr(choice, \"message\", None)\n if message is None and isinstance(choice, dict):\n message = choice.get(\"message\")\n if message is None:\n continue\n\n content = getattr(message, \"content\", None)\n if content is None and isinstance(message, dict):\n content = message.get(\"content\")\n\n if content is not None:\n output.append({\"content\": content})\n\n # 4. Still nothing? Bail out.\n if not output:\n return None\n\n # 5. Existing extraction logic\n for item in output:\n content = item.get(\"content\") if isinstance(item, dict) else getattr(item, \"content\", None)\n if not content:\n continue\n\n # Simple string content\n if isinstance(content, str):\n cleaned = content.strip()\n if cleaned:\n return cleaned\n continue\n\n # List of structured content parts (Responses API or new ChatCompletion v1)\n if isinstance(content, list):\n for part in content:\n # Prefer explicit text fields\n text = part.get(\"text\") if isinstance(part, dict) else getattr(part, \"text\", None)\n if text:\n return text\n\n # Fall back to parsed objects if present\n parsed = part.get(\"parsed\") if isinstance(part, dict) else getattr(part, \"parsed\", None)\n if parsed is not None:\n if hasattr(parsed, \"model_dump_json\"):\n return parsed.model_dump_json()\n if hasattr(parsed, \"dict\"):\n return json.dumps(parsed.dict())\n try:\n return json.dumps(parsed)\n except Exception as exc: # noqa: BLE001\n # Hard-fail rather than silently skipping ambiguous parsed content\n raise RuntimeError(\n f\"Failed to serialize parsed Responses output: {parsed!r}\"\n ) from exc\n\n # Handle unexpected content containers that still expose text/value\n maybe_text = getattr(content, \"text\", None) or getattr(content, \"value\", None)\n if maybe_text:\n return str(maybe_text)\n\n return None\n\n\ndef _extract_text_from_openrouter_kimi(resp_json: dict) -> Optional[str]:\n \"\"\"\n Extract text content from an OpenRouter /chat/completions JSON response\n for Kimi (moonshotai) models, where the payload is a plain dict with\n choices[*].message.content.\n \"\"\"\n if not isinstance(resp_json, dict):\n return None\n\n choices = resp_json.get(\"choices\") or []\n if not choices:\n return None\n\n for choice in choices:\n if not isinstance(choice, dict):\n continue\n message = choice.get(\"message\") or {}\n if not isinstance(message, dict):\n continue\n\n content = message.get(\"content\")\n if not content:\n continue\n\n # Kimi via OpenRouter commonly returns a single JSON string here.\n if isinstance(content, str):\n cleaned = content.strip()\n if cleaned:\n return cleaned\n continue\n\n # Fallback: list of structured parts, e.g. [{\"type\": \"text\", \"text\": \"...\"}]\n if isinstance(content, list):\n parts: list[str] = []\n for part in content:\n if isinstance(part, dict):\n text = part.get(\"text\")\n if isinstance(text, str):\n parts.append(text)\n else:\n inner_content = part.get(\"content\")\n if isinstance(inner_content, str):\n parts.append(inner_content)\n else:\n parts.append(str(part))\n elif isinstance(part, str):\n parts.append(part)\n joined = \"\".join(parts).strip()\n if joined:\n return joined\n\n return None\n\n\n\ndef _extract_text_from_responses_output(resp) -> Optional[str]:\n \"\"\"\n Best-effort extractor for OpenAI Responses API objects.\n Walks output items and returns the first text payload found.\n \"\"\"\n output = getattr(resp, \"output\", None)\n if not output:\n return None\n\n for item in output:\n content = item.get(\"content\") if isinstance(item, dict) else getattr(item, \"content\", None)\n if not content:\n continue\n\n if isinstance(content, str):\n cleaned = content.strip()\n if cleaned:\n return cleaned\n continue\n\n if isinstance(content, list):\n for part in content:\n # Prefer explicit text fields\n text = part.get(\"text\") if isinstance(part, dict) else getattr(part, \"text\", None)\n if text:\n return text\n\n # Fall back to parsed objects if present\n parsed = part.get(\"parsed\") if isinstance(part, dict) else getattr(part, \"parsed\", None)\n if parsed is not None:\n if hasattr(parsed, \"model_dump_json\"):\n return parsed.model_dump_json()\n if hasattr(parsed, \"dict\"):\n return json.dumps(parsed.dict())\n try:\n return json.dumps(parsed)\n except Exception as exc: # noqa: BLE001\n # Hard-fail rather than silently skipping ambiguous parsed content\n raise RuntimeError(\n f\"Failed to serialize parsed Responses output: {parsed!r}\"\n ) from exc\n\n # Handle unexpected content containers that still expose text/value\n maybe_text = getattr(content, \"text\", None) or getattr(content, \"value\", None)\n if maybe_text:\n return str(maybe_text)\n\n return None\n\ndef _raise_if_missing(var: str) -> None:\n if not os.getenv(var):\n raise RuntimeError(f\"Missing environment variable: {var}\")\n\n\ndef _build_openrouter_input_from_prompt(prompt: str) -> list[dict]:\n \"\"\"\n Build a minimal OpenRouter `input` payload for a single\n text-only user prompt, matching the Responses API schema.\n \"\"\"\n return [\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": prompt,\n }\n ],\n }\n ]\n\n\n# --- LLM call dispatcher ---\n@retry(\n reraise=True,\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n retry=retry_if_exception_type(Exception),\n before_sleep=before_sleep_log(logger, logging.WARNING),\n)\ndef call_llm(\n model_name: str,\n prompt: str,\n max_tokens: int = 1200,\n temperature: float = 0.0,\n openrouter: bool = False,\n judge_mode: bool = False,\n):\n \"\"\"\n Unified text-only call.\n\n Returns a tuple (text, reasoning_trace) where:\n - text is the primary string content used by callers.\n - reasoning_trace is a provider-specific raw payload when available\n (DeepSeek/Kimi via OpenRouter), otherwise None.\n \"\"\"\n mn = model_name.strip()\n\n if is_grok_model(mn):\n logger.info(\n \"Calling Grok model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n\n if judge_mode:\n # Using async single call (not pseudo-batched)\n client = _OpenAI(api_key=os.getenv(\"XAI_API_KEY\"), base_url=\"https://api.x.ai/v1\")\n resp = client.chat.completions.create(\n model=mn,\n max_tokens=max_tokens,\n temperature=temperature,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n )\n text = _extract_text_from_responses_output_grok(resp)\n if text:\n return text, None\n raise RuntimeError(f\"Responses API returned no usable text content: {resp!r}\")\n\n text = call_grok_sync(\n model_name=mn,\n prompt=prompt,\n max_tokens=max_tokens,\n temperature=temperature,\n )\n return text, None\n\n # OpenAI gpt-5 family (Responses API)\n if mn.startswith(\"gpt-5\"):\n _raise_if_missing(\"OPENAI_API_KEY\")\n logger.info(\n \"Calling OpenAI Responses API for model '%s' (max_tokens=%d, temperature=1.000).\",\n mn,\n max_tokens,\n )\n client = _OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n\n if judge_mode:\n resp = client.responses.parse(\n model=mn,\n input=[{\"role\": \"user\", \"content\": prompt}],\n temperature=1, # Responses API requires temperature=1 for gpt-5 family\n max_output_tokens=max_tokens,\n top_p=1,\n text_format=JudgeExtraction,\n )\n\n text = _extract_text_from_responses_output(resp)\n if text:\n return text, None\n\n # As a last resort, stringify the whole payload so the caller can attempt to parse\n return str(resp), None\n \n # Eval mode\n resp = client.responses.create(**req)\n if resp.output_text:\n return resp.output_text, None\n\n # If we get here, treat it as a hard failure\n raise RuntimeError(f\"Responses API returned no usable text content: {resp!r}\")\n\n # OpenAI chat models (gpt-4/4o/etc.)\n if mn.startswith(\"gpt-\"):\n _raise_if_missing(\"OPENAI_API_KEY\")\n logger.info(\n \"Calling OpenAI model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n client = _OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n if model_name == \"gpt-4o\":\n resp = client.chat.completions.create(\n model=mn,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=temperature,\n max_tokens=max_tokens,\n top_p=1,\n )\n else:\n resp = client.chat.completions.create(\n model=mn,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=1, # gpt-5 family only supports temperature 1\n max_completion_tokens=max_tokens,\n top_p=1,\n )\n print(repr(resp))\n return resp.choices[0].message.content, None\n\n # Anthropic\n if mn.startswith(\"claude-3\") or mn.startswith(\"claude-opus\"):\n _raise_if_missing(\"ANTHROPIC_API_KEY\")\n logger.info(\n \"Calling Anthropic model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n a = anthropic.Anthropic(api_key=os.getenv(\"ANTHROPIC_API_KEY\"))\n\n if judge_mode:\n from pydantic import BaseModel # type: ignore[redefined-outer-name]\n from anthropic import transform_schema # type: ignore[unused-import]\n\n response = a.beta.messages.create(\n model=\"claude-sonnet-4-5\", #mn,\n betas=[\"structured-outputs-2025-11-13\"],\n max_tokens=max_tokens,\n temperature=temperature,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n output_format={\n \"type\": \"json_schema\",\n \"schema\": transform_schema(JudgeExtraction),\n }\n )\n\n # parsed = JudgeExtraction.model_validate_json(response.text)\n return response.content[0].text, None\n\n msg = a.messages.create(\n model=mn,\n max_tokens=max_tokens,\n temperature=temperature,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n )\n return msg.content[0].text, None\n\n # Google Gemini\n if mn.startswith(\"gemini-\"):\n from google import genai \n from google.genai import types\n\n _raise_if_missing(\"GOOGLE_API_KEY\")\n logger.info(\n \"Calling Gemini model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n\n if judge_mode:\n try:\n client = genai.Client(api_key=os.getenv(\"GOOGLE_API_KEY\")) # type: ignore[name-defined]\n response = client.models.generate_content(\n model=mn,\n contents=prompt,\n config={\n \"temperature\": temperature,\n \"top_p\": 1,\n \"max_output_tokens\": max_tokens,\n \"response_mime_type\": \"application/json\",\n \"response_schema\": JudgeExtraction.model_json_schema(),\n },\n )\n\n parsed = JudgeExtraction.model_validate_json(response.text)\n return parsed.model_dump_json(), None\n\n except Exception as e: # noqa: BLE001\n logger.error(f\"Gemini API call failed: {e}\")\n try:\n if hasattr(response, \"prompt_feedback\"):\n logger.error(f\"Feedback: {response.prompt_feedback}\")\n if hasattr(response, \"candidates\"):\n logger.error(f\"Finish Reason: {response.candidates[0].finish_reason}\")\n except Exception: # noqa: BLE001\n pass\n raise\n\n client = genai.Client(api_key=os.getenv(\"GOOGLE_API_KEY\"))\n response = client.models.generate_content(\n model=mn,\n contents=prompt,\n config={\n \"temperature\": temperature,\n \"top_p\": 1,\n \"max_output_tokens\": max_tokens,\n },\n safety_settings=None,\n )\n return response.text, None\n\n # DeepSeek or Kimi (optionally via OpenRouter)\n mn_lower = mn.lower()\n if mn_lower.startswith(\"deepseek-\") or mn_lower.startswith(\"kimi-\"):\n print(f\" [INFO]: client.py detected {mn_lower}\")\n\n mn_fam = mn_lower.split(\"-\")[0]\n\n if mn_fam == \"kimi\":\n mn_fam = \"moonshotai\"\n\n\n if openrouter:\n print(\" [INFO]: Using Openrouter\")\n _raise_if_missing(\"OPENROUTER_API_KEY\")\n logger.info(\n \"Calling %s model via OpenRouter '%s' (max_tokens=%d, temperature=%.3f).\",\n mn_fam,\n mn,\n max_tokens,\n temperature,\n )\n\n api_key = os.getenv(\"OPENROUTER_API_KEY\")\n\n # Judge mode: request structured JSON with potential reasoning traces.\n if judge_mode:\n print(\"[Info] Using Openrouter with Judge mode\")\n\n # if mn_lower.startswith(\"kimi-\") or mn_lower.startswith(\"deepseek-\"):\n response = requests.post(\n \"https://openrouter.ai/api/v1/chat/completions\",\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n json={\n \"model\": f\"{mn_fam}/{mn}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": prompt,\n }\n ],\n \"response_format\": {\n \"type\": \"json_schema\", \n \"json_schema\": {\n \"name\": \"judge_extraction\",\n \"schema\": JudgeExtraction.schema(),\n \"strict\": True,\n }\n },\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n \"top_p\": 1,\n },\n )\n\n resp_json = response.json()\n\n print(\"[INFO] Debug trace:\")\n print(resp_json)\n\n text = _extract_text_from_openrouter_kimi(resp_json)\n\n\n if text:\n print(text)\n return text, resp_json\n\n print(\"[JUDGE] Error extracting response.\")\n print(\"[INFO] Debug trace:\")\n print(json.dumps(resp_json, indent=4))\n raise RuntimeError(\"Failed to extract text from OpenRouter DeepSeek/Kimi response.\")\n\n print(f\"[INFO] Using model {mn} via OpenRouter\")\n # Non-judge mode over OpenRouter: plain chat completion.\n response = requests.post(\n \"https://openrouter.ai/api/v1/chat/completions\",\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n json={\n \"model\": f\"{mn_fam}/{mn}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": prompt,\n }\n ],\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n \"top_p\": 1,\n },\n )\n\n if response.status_code != 200:\n raise RuntimeError(\n f\"OpenRouter {mn_lower} call failed \"\n f\"(status={response.status_code}): {response.text[:1000]}\"\n )\n\n result = response.json()\n\n try:\n choice = result[\"choices\"][0]\n message = choice.get(\"message\") or {}\n content = message.get(\"content\", \"\")\n if isinstance(content, list):\n parts = []\n for part in content:\n if isinstance(part, dict) and part.get(\"type\") == \"text\":\n parts.append(part.get(\"text\", \"\"))\n else:\n parts.append(str(part))\n content = \"\".join(parts)\n if not isinstance(content, str):\n content = str(content)\n except Exception as exc: # noqa: BLE001\n raise RuntimeError(\n f\"Unexpected OpenRouter response format: {result}\"\n ) from exc\n\n return content, result\n\n # DeepSeek via native API\n if mn_lower.startswith(\"deepseek-\"):\n\n\n\n\n if openrouter:\n print(f\"[INFO] Using model {mn} via OpenRouter\")\n # Non-judge mode over OpenRouter: plain chat completion.\n response = requests.post(\n \"https://openrouter.ai/api/v1/chat/completions\",\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n json={\n \"model\": f\"{mn_fam}/{mn}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": prompt,\n }\n ],\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n \"top_p\": 1,\n },\n )\n\n if response.status_code != 200:\n raise RuntimeError(\n f\"OpenRouter {mn_lower} call failed \"\n f\"(status={response.status_code}): {response.text[:1000]}\"\n )\n\n result = response.json()\n\n try:\n choice = result[\"choices\"][0]\n message = choice.get(\"message\") or {}\n content = message.get(\"content\", \"\")\n if isinstance(content, list):\n parts = []\n for part in content:\n if isinstance(part, dict) and part.get(\"type\") == \"text\":\n parts.append(part.get(\"text\", \"\"))\n else:\n parts.append(str(part))\n content = \"\".join(parts)\n if not isinstance(content, str):\n content = str(content)\n except Exception as exc: # noqa: BLE001\n raise RuntimeError(\n f\"Unexpected OpenRouter response format: {result}\"\n ) from exc\n\n return content, result\n\n\n # Legacy Deepseek API\n _raise_if_missing(\"DEEPSEEK_API_KEY\")\n logger.info(\n \"Calling DeepSeek model '%s' (max_tokens=%d, temperature=%.3f).\",\n mn,\n max_tokens,\n temperature,\n )\n deepseek_model_name = \"deepseek-reasoner\"\n\n api_key = os.getenv(\"DEEPSEEK_API_KEY\")\n\n client = _OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com/v1\", timeout=10)\n\n resp = client.chat.completions.create(\n model=deepseek_model_name,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=temperature,\n max_tokens=max_tokens,\n top_p=1,\n )\n\n print(resp)\n return resp.choices[0].message.content, None\n\n # Placeholder other models\n raise NotImplementedError(\n f\"Model '{mn}' is not wired correctly. Add provider support in quizbench/clients.py.\"\n ) \n\n # Placeholder for other providers\n raise NotImplementedError(\n f\"Model '{mn}' is not wired. Add provider support in quizbench/clients.py.\"\n )\n"}}}} |