1 line
20 KiB
JSON
1 line
20 KiB
JSON
{"repo": "aweussom/tabtabtab", "files": {"archive/enrich-bench.py": {"f86ec4f8071c39d2f80553cc802bad26cc87dab2": {"t": 1785595166.9337513, "n": 19265, "h": "c72a9ef2097781a9", "c": "#!/usr/bin/env python3\n\"\"\"Benchmark multiple Ollama Cloud LLMs against the UG-tab enrichment task.\n\nRuns the same prompts as enrich-private.py against several candidate\nmodels via OpenAI-compatible /chat/completions on the local Ollama\ndaemon (which transparently proxies `:cloud` models to Ollama Cloud\nvia the user's $20/month subscription).\n\nOutputs:\n - `crawler/bench/runs/<model>__<tab>.json` per (model, tab) pair\n - `crawler/bench/COMPARE.md` summary table\n - `crawler/bench/inputs.json` the chosen test inputs\n\nPatterns mirror C:/devel/aweussom/python/evaluator (battle-tested\nOllama integration): temperature 0.7, 600 s timeout, 3-attempt\nexponential backoff with jitter, <think>/<reflection> stripping\nbefore JSON parsing.\n\nTypical use:\n # First run (writes all 4 × 5 = 20 results):\n python crawler/enrich-bench.py\n\n # Resume after Ctrl+C (skips already-cached runs):\n python crawler/enrich-bench.py\n\n # Re-run specific model:\n python crawler/enrich-bench.py --models deepseek-v4-pro:cloud --force\n\n # Regenerate the COMPARE.md from existing run files (no LLM calls):\n python crawler/enrich-bench.py --compare-only\n\"\"\"\nimport argparse\nimport json\nimport re\nimport sys\nimport time\nfrom pathlib import Path\n\ntry:\n import requests\nexcept ImportError:\n print(\"ABORT: pip install requests\", file=sys.stderr)\n sys.exit(1)\n\n# Reuse prompt + parsing helpers from the production pipeline so the\n# bench measures the SAME prompt discipline the proxy + CLI path use.\nsys.path.insert(0, str(Path(__file__).parent))\nfrom enrich import SONG_PROMPT, extract_json, reconfigure_streams\n\n# enrich-private has a hyphen — import via importlib\nimport importlib.util\n_ep_spec = importlib.util.spec_from_file_location(\n \"enrich_private\", Path(__file__).parent / \"enrich-private.py\"\n)\n_ep = importlib.util.module_from_spec(_ep_spec)\n_ep_spec.loader.exec_module(_ep)\nSTRICT_SUFFIX = _ep.STRICT_SUFFIX\nsalvage_truncated_json = _ep.salvage_truncated_json\nslugify = _ep.slugify\n\n# ---------------------------------------------------------------------------\n# Config\n# ---------------------------------------------------------------------------\n\nDEFAULT_MODELS = [\n \"deepseek-v4-flash:cloud\",\n \"deepseek-v4-pro:cloud\",\n \"nemotron-3-super:cloud\",\n \"qwen3.5:cloud\",\n]\n\n# Test tabs: 5 English songs picking different difficulty axes. Lyrics\n# bodies are read from the actual UG import so hallucination scoring\n# (key_phrases ∈ body) measures real signal.\nTEST_TAB_KEYS = [\n \"Jolene\", # famous, classic country — expected easy\n \"Tecumseh Valley\", # obscure, has a \"John Prine vs Townes Van Zandt\" attribution trap\n \"Let Her Go\", # modern pop (Passenger, 2012)\n \"Hallelujah\", # famous cover (Buckley) — tests cover-vs-original conflation\n \"Lady In Black\", # older obscure rock (Uriah Heep, 1971)\n]\n\nOLLAMA_BASE = \"http://localhost:11434/v1\"\n\nTEMPERATURE = 0.7\nTIMEOUT_S = 600\nRETRY_ATTEMPTS = 3\nRETRY_BACKOFF_BASE_S = 5\nRETRY_BACKOFF_MAX_S = 60\n\nSYSTEM_MSG = (\n \"You enrich a guitar-tab catalog with search metadata for a web app. \"\n \"Output ONE JSON object only — no markdown fences, no commentary, no \"\n \"surrounding text.\"\n)\n\n# ---------------------------------------------------------------------------\n# LLM call (mirrors evaluator's patterns: retry/backoff + think-strip)\n# ---------------------------------------------------------------------------\n\n_THINK_RE = re.compile(r\"(?:<think>)?[\\s\\S]*?</think>\\s*\", re.IGNORECASE)\n_REFLECT_RE = re.compile(r\"<reflection>[\\s\\S]*?</reflection>\\s*\", re.IGNORECASE)\n_FENCE_OPEN_RE = re.compile(r\"^```(?:json)?\\s*\", re.IGNORECASE)\n_FENCE_CLOSE_RE = re.compile(r\"\\s*```\\s*$\")\n\n\ndef strip_reasoning(text):\n \"\"\"Remove <think>/<reflection> blocks from LLM output before JSON parse.\"\"\"\n text = _THINK_RE.sub(\"\", text or \"\")\n text = _REFLECT_RE.sub(\"\", text)\n return text.strip()\n\n\ndef strip_fences(text):\n \"\"\"Remove ```json ... ``` wrapping if the model added it despite the prompt.\"\"\"\n text = _FENCE_OPEN_RE.sub(\"\", text)\n text = _FENCE_CLOSE_RE.sub(\"\", text)\n return text.strip()\n\n\ndef call_ollama(model, system_msg, user_msg, temperature=TEMPERATURE,\n timeout=TIMEOUT_S, base_url=None, api_key=None, extra_body=None):\n \"\"\"OpenAI-compat /chat/completions. Returns dict:\n {content, raw, latency_ms, tokens_in, tokens_out, error}\n On final failure (after retries), `content` is None and `error` is set.\n\n `base_url` defaults to local Ollama; pass e.g. https://api.xiaomimimo.com/v1\n for a hosted provider. `api_key` adds an Authorization: Bearer header\n (local Ollama needs none; Mimo / Ollama Cloud require one). `extra_body`\n is a dict merged into the request payload — e.g.\n {\"thinking\": {\"type\": \"disabled\"}} to turn off MiMo's reasoning mode.\n \"\"\"\n base = (base_url or OLLAMA_BASE).rstrip(\"/\")\n url = f\"{base}/chat/completions\"\n headers = {\"Content-Type\": \"application/json\"}\n if api_key:\n headers[\"Authorization\"] = f\"Bearer {api_key}\"\n payload = {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": system_msg},\n {\"role\": \"user\", \"content\": user_msg},\n ],\n \"stream\": False,\n \"temperature\": temperature,\n }\n if extra_body:\n payload.update(extra_body)\n last_err = None\n t0 = time.time()\n for attempt in range(1, RETRY_ATTEMPTS + 1):\n try:\n resp = requests.post(url, json=payload, headers=headers, timeout=timeout)\n if resp.status_code != 200:\n raise RuntimeError(f\"HTTP {resp.status_code}: {resp.text[:200]}\")\n data = resp.json()\n latency_ms = int((time.time() - t0) * 1000)\n content = (data.get(\"choices\") or [{}])[0].get(\"message\", {}).get(\"content\", \"\")\n usage = data.get(\"usage\") or {}\n return {\n \"content\": content,\n \"raw\": data,\n \"latency_ms\": latency_ms,\n \"tokens_in\": usage.get(\"prompt_tokens\"),\n \"tokens_out\": usage.get(\"completion_tokens\"),\n \"error\": None,\n }\n except Exception as err:\n last_err = err\n if attempt >= RETRY_ATTEMPTS:\n break\n delay = min(RETRY_BACKOFF_BASE_S * (2 ** (attempt - 1)),\n RETRY_BACKOFF_MAX_S)\n jitter = delay * 0.1\n wait_s = delay + (jitter * (time.time() % 1))\n print(f\" attempt {attempt}/{RETRY_ATTEMPTS} failed: {err} — \"\n f\"retrying in {wait_s:.1f}s\", file=sys.stderr)\n time.sleep(wait_s)\n return {\n \"content\": None,\n \"raw\": None,\n \"latency_ms\": int((time.time() - t0) * 1000),\n \"tokens_in\": None,\n \"tokens_out\": None,\n \"error\": str(last_err),\n }\n\n\n# ---------------------------------------------------------------------------\n# Hallucination scoring\n# ---------------------------------------------------------------------------\n\ndef _fold(s):\n \"\"\"Loose token-equality fold: lowercase, strip diacritics + punctuation.\"\"\"\n s = (s or \"\").lower()\n s = re.sub(r\"[^\\w\\s]\", \" \", s, flags=re.UNICODE)\n s = re.sub(r\"\\s+\", \" \", s).strip()\n return s\n\n\ndef key_phrases_in_body(phrases, body):\n \"\"\"Fraction of key_phrases that appear (verbatim or token-folded) in body.\n Returns (hits, total, fraction).\"\"\"\n if not phrases:\n return 0, 0, 1.0 # no claim = no hallucination\n body_folded = _fold(body)\n hits = 0\n for p in phrases:\n p_folded = _fold(str(p))\n if not p_folded:\n continue\n if p_folded in body_folded:\n hits += 1\n return hits, len(phrases), (hits / len(phrases)) if phrases else 1.0\n\n\n# ---------------------------------------------------------------------------\n# Run orchestration\n# ---------------------------------------------------------------------------\n\ndef pick_test_tabs(import_path, keys):\n \"\"\"Find the named tabs in the UG import; return [{artist, song, body, id}].\"\"\"\n data = json.loads(Path(import_path).read_text(encoding=\"utf-8\"))\n tabs = data.get(\"tabs\") or []\n picks = {}\n for t in tabs:\n song = t.get(\"song\") or \"\"\n for k in keys:\n if k in song and k not in picks:\n picks[k] = {\n \"key\": k,\n \"artist\": t.get(\"artist\", \"\"),\n \"song\": song,\n \"body\": t.get(\"body\") or \"\", # full body — Nemotron flat-rate makes token cost irrelevant\n \"id\": f\"{slugify(t.get('artist', ''))}__{slugify(song)}\",\n }\n break\n missing = [k for k in keys if k not in picks]\n if missing:\n print(f\"WARN: didn't find these test tabs in import: {missing}\",\n file=sys.stderr)\n return [picks[k] for k in keys if k in picks]\n\n\ndef run_one(model, tab, base_url=None, api_key=None, extra_body=None):\n \"\"\"Run a single (model, tab) pair. Returns the full per-run dict.\"\"\"\n user_msg = SONG_PROMPT.format(\n artist=tab[\"artist\"], song=tab[\"song\"], body=tab[\"body\"]\n ) + STRICT_SUFFIX\n\n llm = call_ollama(model, SYSTEM_MSG, user_msg, base_url=base_url,\n api_key=api_key, extra_body=extra_body)\n parsed = None\n parse_ok = False\n parse_path = None # \"strict\" | \"salvaged\" | \"failed\"\n\n if llm[\"content\"]:\n cleaned = strip_fences(strip_reasoning(llm[\"content\"]))\n try:\n parsed = extract_json(cleaned)\n parse_ok = parsed is not None\n parse_path = \"strict\" if parse_ok else None\n except Exception:\n parsed = None\n parse_ok = False\n if not parse_ok:\n salvaged = salvage_truncated_json(cleaned)\n if salvaged is not None:\n parsed = salvaged\n parse_ok = True\n parse_path = \"salvaged\"\n if not parse_ok:\n parse_path = \"failed\"\n\n key_phrases = (parsed or {}).get(\"key_phrases\") or []\n hits, total, frac = key_phrases_in_body(key_phrases, tab[\"body\"])\n\n return {\n \"model\": model,\n \"tab_key\": tab[\"key\"],\n \"tab_id\": tab[\"id\"],\n \"artist\": tab[\"artist\"],\n \"song\": tab[\"song\"],\n \"latency_ms\": llm[\"latency_ms\"],\n \"tokens_in\": llm[\"tokens_in\"],\n \"tokens_out\": llm[\"tokens_out\"],\n \"error\": llm[\"error\"],\n \"raw_content\": llm[\"content\"],\n \"parse_ok\": parse_ok,\n \"parse_path\": parse_path,\n \"enrichment\": parsed,\n \"key_phrases_hit\": hits,\n \"key_phrases_total\": total,\n \"key_phrases_fraction\": round(frac, 3),\n }\n\n\ndef run_key(model, tab_id):\n \"\"\"Filesystem-safe key for `crawler/bench/runs/<model>__<tab>.json`.\"\"\"\n safe_model = model.replace(\":\", \"_\").replace(\"/\", \"_\")\n return f\"{safe_model}__{tab_id}\"\n\n\n# ---------------------------------------------------------------------------\n# Compare-table renderer\n# ---------------------------------------------------------------------------\n\ndef render_compare_md(runs_dir, models, tabs):\n \"\"\"Build COMPARE.md from existing run files.\"\"\"\n by_model = {m: {} for m in models}\n for f in sorted(runs_dir.glob(\"*.json\")):\n try:\n run = json.loads(f.read_text(encoding=\"utf-8\"))\n except Exception:\n continue\n m = run[\"model\"]\n tid = run[\"tab_id\"]\n if m in by_model:\n by_model[m][tid] = run\n\n out = []\n out.append(\"# Enrichment Bench — Ollama Cloud comparison\")\n out.append(\"\")\n out.append(f\"Models tested: {', '.join(f'`{m}`' for m in models)}\")\n out.append(\"\")\n out.append(f\"Tabs tested ({len(tabs)}):\")\n for tab in tabs:\n out.append(f\"- **{tab['key']}** — {tab['artist']} — {tab['song']} \"\n f\"(id `{tab['id']}`)\")\n out.append(\"\")\n\n # Per-tab tables\n out.append(\"## Per-tab results\")\n out.append(\"\")\n for tab in tabs:\n out.append(f\"### {tab['key']} — {tab['artist']}\")\n out.append(\"\")\n out.append(\"| Model | parse | latency | tokens (in/out) | kp hit/total | \"\n \"language | themes | key_phrases (first 2) |\")\n out.append(\"|---|---|---|---|---|---|---|---|\")\n for m in models:\n run = by_model.get(m, {}).get(tab[\"id\"])\n if not run:\n out.append(f\"| `{m}` | — | — | — | — | — | — | _(no run)_ |\")\n continue\n parse = \"✓\" if run[\"parse_ok\"] else \"✗\"\n if run[\"parse_path\"] == \"salvaged\":\n parse = \"✓ (salvaged)\"\n lat = f\"{run['latency_ms']/1000:.1f}s\"\n ti, to = run[\"tokens_in\"], run[\"tokens_out\"]\n toks = f\"{ti}/{to}\" if ti is not None and to is not None else \"—\"\n kp = f\"{run['key_phrases_hit']}/{run['key_phrases_total']}\"\n enrich = run[\"enrichment\"] or {}\n lang = enrich.get(\"language\", \"—\")\n themes = \", \".join((enrich.get(\"themes\") or [])[:3]) or \"—\"\n phrases = (enrich.get(\"key_phrases\") or [])[:2]\n phrases_str = \" / \".join(f\"_{p}_\" for p in phrases) or \"—\"\n err_note = f\" ⚠ {run['error']}\" if run[\"error\"] else \"\"\n out.append(f\"| `{m}` | {parse}{err_note} | {lat} | {toks} | \"\n f\"{kp} | {lang} | {themes} | {phrases_str} |\")\n out.append(\"\")\n\n # Aggregate stats\n out.append(\"## Aggregate (across all tabs)\")\n out.append(\"\")\n out.append(\"| Model | parse rate | mean latency | mean tokens out | \"\n \"mean kp hit-rate |\")\n out.append(\"|---|---|---|---|---|\")\n for m in models:\n runs = list(by_model.get(m, {}).values())\n if not runs:\n out.append(f\"| `{m}` | — | — | — | — |\")\n continue\n n = len(runs)\n parse_n = sum(1 for r in runs if r[\"parse_ok\"])\n lat_avg_s = sum(r[\"latency_ms\"] for r in runs) / n / 1000\n toks_out = [r[\"tokens_out\"] for r in runs if r[\"tokens_out\"]]\n toks_avg = (sum(toks_out) / len(toks_out)) if toks_out else None\n kp_fracs = [r[\"key_phrases_fraction\"] for r in runs if r[\"parse_ok\"]]\n kp_avg = (sum(kp_fracs) / len(kp_fracs)) if kp_fracs else None\n\n parse_cell = f\"{parse_n}/{n} ({100*parse_n/n:.0f}%)\"\n lat_cell = f\"{lat_avg_s:.1f}s\"\n toks_cell = f\"{toks_avg:.0f}\" if toks_avg is not None else \"—\"\n kp_cell = f\"{kp_avg*100:.0f}%\" if kp_avg is not None else \"—\"\n out.append(f\"| `{m}` | {parse_cell} | {lat_cell} | {toks_cell} | {kp_cell} |\")\n out.append(\"\")\n out.append(\"**Read this table as**: parse-rate is the \"\n \"first quality gate (must be ~100%); key_phrases hit-rate is \"\n \"the hallucination signal (lower = more invented quotes); \"\n \"latency matters most for batch enrichment of large UG imports.\")\n return \"\\n\".join(out)\n\n\n# ---------------------------------------------------------------------------\n# Main\n# ---------------------------------------------------------------------------\n\ndef parse_args():\n p = argparse.ArgumentParser()\n p.add_argument(\"--input\", default=\"crawler/private/ug-import.json\")\n p.add_argument(\"--out-dir\", default=\"crawler/bench\")\n p.add_argument(\"--models\", nargs=\"*\", default=DEFAULT_MODELS)\n p.add_argument(\"--base\", default=OLLAMA_BASE,\n help=f\"OpenAI-compat base URL (default: {OLLAMA_BASE}). \"\n \"Use https://api.mimo.xiaomi.com/v1 for Mimo, \"\n \"https://ollama.com/v1 for Ollama Cloud direct, etc.\")\n p.add_argument(\"--api-key-env\", default=None,\n help=\"Name of an env var holding the Bearer API key. \"\n \"Omit for local Ollama (no auth). E.g. MIMO_API_KEY.\")\n p.add_argument(\"--extra-body\", default=None,\n help=\"JSON string merged into the request payload. \"\n \"E.g. '{\\\"thinking\\\": {\\\"type\\\": \\\"disabled\\\"}}' to \"\n \"turn off MiMo's reasoning mode.\")\n p.add_argument(\"--force\", action=\"store_true\",\n help=\"Re-run runs even if a cached file exists\")\n p.add_argument(\"--compare-only\", action=\"store_true\",\n help=\"Skip LLM calls, just regenerate COMPARE.md\")\n return p.parse_args()\n\n\ndef main():\n reconfigure_streams()\n args = parse_args()\n out_dir = Path(args.out_dir)\n runs_dir = out_dir / \"runs\"\n runs_dir.mkdir(parents=True, exist_ok=True)\n\n # Resolve the API key from the named env var, if any. Never logged.\n import os\n api_key = os.environ.get(args.api_key_env) if args.api_key_env else None\n if args.api_key_env and not api_key:\n print(f\"ABORT: --api-key-env {args.api_key_env} is set but the env \"\n f\"var is empty/unset.\", file=sys.stderr)\n return 1\n\n extra_body = None\n if args.extra_body:\n try:\n extra_body = json.loads(args.extra_body)\n except json.JSONDecodeError as e:\n print(f\"ABORT: --extra-body is not valid JSON: {e}\", file=sys.stderr)\n return 1\n\n tabs = pick_test_tabs(args.input, TEST_TAB_KEYS)\n if not tabs:\n print(f\"ABORT: no test tabs found in {args.input}\", file=sys.stderr)\n return 1\n\n # Persist the inputs so a reader can see exactly what was tested\n (out_dir / \"inputs.json\").write_text(\n json.dumps(tabs, indent=2, ensure_ascii=False), encoding=\"utf-8\"\n )\n\n if args.compare_only:\n md = render_compare_md(runs_dir, args.models, tabs)\n (out_dir / \"COMPARE.md\").write_text(md, encoding=\"utf-8\")\n print(f\"Wrote {out_dir / 'COMPARE.md'} (compare-only).\")\n return 0\n\n total = len(args.models) * len(tabs)\n done = 0\n auth_note = f\" (auth via ${args.api_key_env})\" if args.api_key_env else \"\"\n print(f\"Running {total} (model × tab) combinations against {args.base}{auth_note}\")\n for model in args.models:\n for tab in tabs:\n done += 1\n key = run_key(model, tab[\"id\"])\n run_path = runs_dir / f\"{key}.json\"\n if run_path.exists() and not args.force:\n print(f\"[{done:2}/{total}] cached: {model} × {tab['key']}\")\n continue\n print(f\"[{done:2}/{total}] {model} × {tab['key']} ...\", flush=True)\n result = run_one(model, tab, base_url=args.base, api_key=api_key,\n extra_body=extra_body)\n run_path.write_text(\n json.dumps(result, indent=2, ensure_ascii=False),\n encoding=\"utf-8\"\n )\n status = \"OK\" if result[\"parse_ok\"] else \"FAIL\"\n if result[\"error\"]:\n status = f\"ERROR ({result['error'][:60]})\"\n kp = f\"{result['key_phrases_hit']}/{result['key_phrases_total']}\"\n print(f\" → {status} | {result['latency_ms']/1000:.1f}s | \"\n f\"kp {kp}\")\n\n md = render_compare_md(runs_dir, args.models, tabs)\n (out_dir / \"COMPARE.md\").write_text(md, encoding=\"utf-8\")\n print(f\"\\nDone. See {out_dir / 'COMPARE.md'}.\")\n return 0\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n"}}}} |