{"repo": "aihubmix/model-showdown", "files": {"showdown.config.json": {"65d4b5a87927aef2b7116e97f02aa3dde934fb19": {"t": 1785768970.4413335, "n": 425, "h": "e795539de08345a5", "c": "{\n \"gateway\": \"https://aihubmix.com/v1/chat/completions\",\n \"api_key_env\": \"AIHUBMIX_API_KEY\",\n \"brand\": {\n \"tagline\": \"AIHubMix: Unified API, 800+ models\",\n \"watermark\": \"Made with model-showdown \\u00b7 aihubmix.com\",\n \"gallery\": \"https://aihubmix.github.io/model-showdown/\"\n },\n \"web\": {\n \"server_domain\": \"https://aihubmix.com\",\n \"caption_model\": \"qwen3-vl-flash\"\n },\n \"models\": {},\n \"pricing\": {}\n}\n"}}, "run_showdown.py": {"65d4b5a87927aef2b7116e97f02aa3dde934fb19": {"t": 1785768995.1336634, "n": 41694, "h": "fed8c2c3a7b6afda", "c": "#!/usr/bin/env python3\n\"\"\"One-shot model showdown: same prompt -> N models via aihubmix chat/completions.\n\nUsage: python3 run_showdown.py [--models m1,m2,...] [--record SECONDS]\nReads /task.md, writes work_/index.html, raw_.json, metrics.json\n\n- Streams (SSE) with a 30s heartbeat log — long-thinking models (Kimi K3) get their\n connection killed by upstream proxies if no bytes flow for ~10-15 min.\n- metrics.json is written incrementally after EACH model finishes (a killed run\n loses nothing).\n- --record N pipelines recording: each model's artifact is recorded for N seconds\n as soon as its generation finishes, overlapping the slower models' generation.\n\"\"\"\nimport argparse\nimport base64\nimport concurrent.futures as cf\nimport glob as globmod\nimport json\nimport os\nimport re\nimport subprocess\nimport sys\nimport threading\nimport time\nimport urllib.request\nfrom datetime import datetime, timezone\n\nSCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))\n\n# showdown.config.json externalizes gateway/brand/model overrides so outside\n# users can self-deploy BYOK; when absent, fall back to built-in defaults so\n# the old workflow is unaffected.\nCONFIG_PATH = os.path.join(SCRIPT_DIR, \"showdown.config.json\")\nCONFIG = {}\nif os.path.exists(CONFIG_PATH):\n with open(CONFIG_PATH) as _f:\n CONFIG = json.load(_f)\n\nGATEWAY = os.environ.get(\"SHOWDOWN_GATEWAY\") \\\n or CONFIG.get(\"gateway\", \"https://aihubmix.com/v1/chat/completions\")\nAPI_KEY_ENV = CONFIG.get(\"api_key_env\", \"AIHUBMIX_API_KEY\")\n# Key validation lives in main(), not at import time: callers like the webapp\n# must be able to import MODELS/PRICING without a key (BYOK keys are passed to\n# the subprocess per-job via environment variables).\nAPI_KEY = os.environ.get(API_KEY_ENV)\n\n\ndef auth_headers(kind):\n \"\"\"kind: openai / anthropic / google — send the key header matching the protocol.\"\"\"\n if kind == \"anthropic\":\n return {\"x-api-key\": API_KEY}\n if kind == \"google\":\n return {\"x-goog-api-key\": API_KEY}\n return {\"Authorization\": f\"Bearer {API_KEY}\"}\n\n# USD per 1M tokens (input, output)\nPRICING = {\n \"kimi-k3\": (3.0, 15.0),\n \"claude-opus-4-8\": (5.0, 25.0),\n \"claude-opus-4-8-think\": (5.0, 25.0),\n \"claude-fable-5\": (10.0, 50.0),\n \"gpt-5.6-sol\": (5.0, 30.0),\n # qwen3.8-max-preview has no official per-token list price (subscription\n # preview); estimated from the previous-gen qwen3.7-max list price for now\n \"qwen3.8-max-preview\": (1.25, 3.75),\n \"gemini-3.6-flash\": (1.50, 7.50),\n}\n\n# display name, max_tokens override (K3 always thinks at max effort — reasoning\n# eats the budget). No fallback channels: the badge must reflect the real model\n# (coding-* channels bill differently and would misrepresent identity/cost).\n# Default lineup = every model at its MAX reasoning mode:\n# kimi-k3 always reasons at max effort (not configurable)\n# claude-opus-4-8-think is the thinking variant of Opus 4.8\n# gpt-5.6-sol gets reasoning_effort=max, its top tier (verified effective on chat/completions)\n# \"params\" is merged verbatim into the request payload.\nMODELS = {\n \"kimi-k3\": {\"display\": \"Kimi K3\", \"max_tokens\": 160000, \"retries\": 12},\n \"claude-opus-4-8-think\": {\"display\": \"Claude Opus 4.8\", \"max_tokens\": 100000},\n # effort is ONLY effective via /v1/responses — chat/completions silently\n # drops reasoning_effort for this model (verified by 3x3 A/B test)\n \"gpt-5.6-sol\": {\"display\": \"GPT-5.6 Sol\", \"max_tokens\": 100000,\n \"endpoint\": \"responses\",\n \"params\": {\"reasoning\": {\"effort\": \"max\"}}},\n # non-thinking / substitute / guest contestants — NOT in the default lineup,\n # pick explicitly via --models\n \"claude-opus-4-8\": {\"display\": \"Claude Opus 4.8\", \"lineup\": False},\n # Fable 5: adaptive reasoning via native /v1/messages (NOT chat/completions).\n # 128000 is claude-fable-5's hard ceiling on the gateway (verified via\n # direct curl 400). max_tokens must always be set explicitly — omitting it\n # does NOT fall back to that ceiling, it silently drops to ~4096, which the\n # thinking budget blows through instantly (finish_reason=length, empty\n # content).\n # endpoint=messages: switched from chat/completions hoping incremental\n # content_block_delta events would keep the wire hot — this did NOT fix\n # anything, both endpoints behave identically for this model/gateway, and\n # neither is worse than the other. Kept endpoint=messages since it's the\n # architecturally correct protocol.\n # ROOT CAUSE (confirmed via AIHubMix backend tid export, ep04): for a\n # real task (ref image + long prompt) the model needs ~12-13min before\n # its *first* content byte reaches the gateway (all of it is silent\n # server-side \"thinking\" — no partial frames, on either endpoint). Some\n # intermediary (proxy/LB — ours or AIHubMix's) kills idle client\n # connections at ~10-11min, i.e. 1-2min *before* that first byte would\n # arrive. This is NOT proportional to budget_tokens — 100k/40k/18k budgets\n # all died at the same ~10-11min mark, and the real run still used 47k\n # thinking_tokens despite budget_tokens=18000 (budget_tokens is a soft\n # hint here, not a hard cap). So lowering the budget doesn't help; the\n # generation itself still needs the same wall-clock time regardless.\n # reasoning:{effort:\"max\",display:\"summarized\"} was also tried as a\n # workaround: 400 invalid field on /v1/messages; on /v1/chat/completions\n # the request never even registered on the AIHubMix dashboard — dead end.\n # WORKAROUND when this happens: the generation still completes\n # successfully server-side even though the client connection dies —\n # ask the user to pull the AIHubMix dashboard record for the request's\n # tid (shown in error bodies, or found via the dashboard's request log)\n # and export it; the complete response (incl. the ```html block) can be\n # recovered from there and written directly to work_/index.html,\n # no need to re-run generation. See episodes/ep04's recovery for tid\n # 2026072014131970707025835020540.\n \"claude-fable-5\": {\"display\": \"Claude Fable 5\", \"max_tokens\": 128000, \"lineup\": False,\n \"endpoint\": \"messages\",\n \"params\": {\"thinking\": {\"type\": \"enabled\", \"budget_tokens\": 40000}}},\n # Qwen3.8 Max Preview: generous output budget for large HTML artifacts;\n # opt in explicitly via --models\n \"qwen3.8-max-preview\": {\"display\": \"Qwen3.8 Max\", \"max_tokens\": 120000, \"lineup\": False},\n # Gemini 3.6 Flash (released 2026-07-21): native Google generateContent\n # protocol via the gateway's /gemini passthrough (x-goog-api-key auth,\n # SSE candidates[].content.parts[].text deltas). 65536 is its hard output\n # ceiling per the official API spec.\n \"gemini-3.6-flash\": {\"display\": \"Gemini 3.6 Flash\", \"max_tokens\": 65536, \"lineup\": False,\n \"endpoint\": \"gemini\",\n \"params\": {\"generationConfig\": {\"maxOutputTokens\": 65536,\n \"thinkingConfig\": {\"thinkingLevel\": \"high\"}}}},\n}\n# models/pricing in the config override/extend the built-in tables (a matching\n# key replaces the whole entry)\nPRICING.update({k: tuple(v) for k, v in CONFIG.get(\"pricing\", {}).items()})\nMODELS.update(CONFIG.get(\"models\", {}))\n# Ad-hoc models the webapp picks dynamically from the model catalog endpoint\n# (/api/v1/models) are injected via env and only apply to this process\nMODELS.update(json.loads(os.environ.get(\"SHOWDOWN_EXTRA_MODELS\") or \"{}\"))\nPRICING.update({k: tuple(v) for k, v in\n json.loads(os.environ.get(\"SHOWDOWN_EXTRA_PRICING\") or \"{}\").items()})\nDEFAULT_MAX_TOKENS = 60000\n\nREQUEST_TIMEOUT_S = 3600 # per socket read while streaming\n# Used to be 1200s: with reasoning=max + large multimodal input (images), real\n# generation time reliably exceeds 30 minutes (TTFT alone is 11-12 min), so at\n# 1200s the CLIENT would preemptively cut the connection while the server was\n# still streaming fine — not a gateway/upstream problem, the value was too low.\nRETRIES = 3\nRETRY_WAIT_S = 60\nHEARTBEAT_S = 30\n\n\ndef sniff_image_mime(data, ext):\n \"\"\"Determine mime from the actual bytes first, not the file extension.\n Anthropic (claude) strictly checks that the declared media type matches\n the real bytes — a png/jpeg mismatch is an immediate 400; a .png file\n holding jpeg bytes once wasted a whole generation round and forced a\n restart. The extension is only a last-resort fallback.\"\"\"\n if data[:3] == b\"\\xff\\xd8\\xff\":\n return \"jpeg\"\n if data[:8] == b\"\\x89PNG\\r\\n\\x1a\\n\":\n return \"png\"\n if data[:4] == b\"RIFF\" and data[8:12] == b\"WEBP\":\n return \"webp\"\n if data[:6] in (b\"GIF87a\", b\"GIF89a\"):\n return \"gif\"\n return {\"jpg\": \"jpeg\", \"jpeg\": \"jpeg\", \"png\": \"png\", \"webp\": \"webp\",\n \"gif\": \"gif\"}.get(ext, \"png\")\n\n\ndef build_user_content(prompt, ep_dir):\n \"\"\"Text-only content, or multimodal [text + image_url] when the episode\n ships a reference image (ref.png / ref.jpg).\"\"\"\n refs = sorted(r for r in globmod.glob(f\"{ep_dir}/ref*\")\n if r.rsplit(\".\", 1)[-1].lower() in (\"png\", \"jpg\", \"jpeg\", \"webp\", \"gif\"))\n if not refs:\n return prompt\n parts = [{\"type\": \"text\", \"text\": prompt}]\n for ref in refs:\n ext = ref.rsplit(\".\", 1)[-1].lower()\n with open(ref, \"rb\") as f:\n raw = f.read()\n mime = sniff_image_mime(raw, ext)\n b64 = base64.b64encode(raw).decode()\n parts.append({\"type\": \"image_url\",\n \"image_url\": {\"url\": f\"data:image/{mime};base64,{b64}\"}})\n return parts\n\n\ndef to_responses_input(content):\n \"\"\"Convert chat-style user content into Responses-API input parts.\"\"\"\n if isinstance(content, str):\n return content\n parts = []\n for p in content:\n if p[\"type\"] == \"text\":\n parts.append({\"type\": \"input_text\", \"text\": p[\"text\"]})\n elif p[\"type\"] == \"image_url\":\n parts.append({\"type\": \"input_image\", \"image_url\": p[\"image_url\"][\"url\"]})\n return [{\"role\": \"user\", \"content\": parts}]\n\n\ndef to_messages_content(content):\n \"\"\"Convert chat-style user content (text + image_url) into Anthropic\n Messages API native content blocks (text + image/base64).\"\"\"\n if isinstance(content, str):\n return content\n parts = []\n for p in content:\n if p[\"type\"] == \"text\":\n parts.append({\"type\": \"text\", \"text\": p[\"text\"]})\n elif p[\"type\"] == \"image_url\":\n header, b64 = p[\"image_url\"][\"url\"].split(\",\", 1)\n media_type = header.split(\":\")[1].split(\";\")[0]\n parts.append({\"type\": \"image\",\n \"source\": {\"type\": \"base64\", \"media_type\": media_type, \"data\": b64}})\n return parts\n\n\ndef to_gemini_parts(content):\n \"\"\"Convert chat-style user content (text + image_url) into Gemini native\n generateContent `parts` (text / inline_data base64).\"\"\"\n if isinstance(content, str):\n return [{\"text\": content}]\n parts = []\n for p in content:\n if p[\"type\"] == \"text\":\n parts.append({\"text\": p[\"text\"]})\n elif p[\"type\"] == \"image_url\":\n header, b64 = p[\"image_url\"][\"url\"].split(\",\", 1)\n mime_type = header.split(\":\")[1].split(\";\")[0]\n parts.append({\"inline_data\": {\"mime_type\": mime_type, \"data\": b64}})\n return parts\n\n\ndef _iso_now():\n return datetime.now(timezone.utc).isoformat()\n\n\nclass Timeline:\n \"\"\"Per-request client-side event timeline (UTC timestamps), written\n incrementally to ep_dir/timeline_