{"repo": "aweussom/tabtabtab", "files": {"archive/enrich-bench.py": {"f86ec4f8071c39d2f80553cc802bad26cc87dab2": {"t": 1785595166.9337513, "n": 19265, "h": "c72a9ef2097781a9", "c": "#!/usr/bin/env python3\n\"\"\"Benchmark multiple Ollama Cloud LLMs against the UG-tab enrichment task.\n\nRuns the same prompts as enrich-private.py against several candidate\nmodels via OpenAI-compatible /chat/completions on the local Ollama\ndaemon (which transparently proxies `:cloud` models to Ollama Cloud\nvia the user's $20/month subscription).\n\nOutputs:\n  - `crawler/bench/runs/<model>__<tab>.json` per (model, tab) pair\n  - `crawler/bench/COMPARE.md`              summary table\n  - `crawler/bench/inputs.json`             the chosen test inputs\n\nPatterns mirror C:/devel/aweussom/python/evaluator (battle-tested\nOllama integration): temperature 0.7, 600 s timeout, 3-attempt\nexponential backoff with jitter, <think>/<reflection> stripping\nbefore JSON parsing.\n\nTypical use:\n    # First run (writes all 4 × 5 = 20 results):\n    python crawler/enrich-bench.py\n\n    # Resume after Ctrl+C (skips already-cached runs):\n    python crawler/enrich-bench.py\n\n    # Re-run specific model:\n    python crawler/enrich-bench.py --models deepseek-v4-pro:cloud --force\n\n    # Regenerate the COMPARE.md from existing run files (no LLM calls):\n    python crawler/enrich-bench.py --compare-only\n\"\"\"\nimport argparse\nimport json\nimport re\nimport sys\nimport time\nfrom pathlib import Path\n\ntry:\n    import requests\nexcept ImportError:\n    print(\"ABORT: pip install requests\", file=sys.stderr)\n    sys.exit(1)\n\n# Reuse prompt + parsing helpers from the production pipeline so the\n# bench measures the SAME prompt discipline the proxy + CLI path use.\nsys.path.insert(0, str(Path(__file__).parent))\nfrom enrich import SONG_PROMPT, extract_json, reconfigure_streams\n\n# enrich-private has a hyphen — import via importlib\nimport importlib.util\n_ep_spec = importlib.util.spec_from_file_location(\n    \"enrich_private\", Path(__file__).parent / \"enrich-private.py\"\n)\n_ep = importlib.util.module_from_spec(_ep_spec)\n_ep_spec.loader.exec_module(_ep)\nSTRICT_SUFFIX = _ep.STRICT_SUFFIX\nsalvage_truncated_json = _ep.salvage_truncated_json\nslugify = _ep.slugify\n\n# ---------------------------------------------------------------------------\n# Config\n# ---------------------------------------------------------------------------\n\nDEFAULT_MODELS = [\n    \"deepseek-v4-flash:cloud\",\n    \"deepseek-v4-pro:cloud\",\n    \"nemotron-3-super:cloud\",\n    \"qwen3.5:cloud\",\n]\n\n# Test tabs: 5 English songs picking different difficulty axes. Lyrics\n# bodies are read from the actual UG import so hallucination scoring\n# (key_phrases ∈ body) measures real signal.\nTEST_TAB_KEYS = [\n    \"Jolene\",            # famous, classic country — expected easy\n    \"Tecumseh Valley\",   # obscure, has a \"John Prine vs Townes Van Zandt\" attribution trap\n    \"Let Her Go\",        # modern pop (Passenger, 2012)\n    \"Hallelujah\",        # famous cover (Buckley) — tests cover-vs-original conflation\n    \"Lady In Black\",     # older obscure rock (Uriah Heep, 1971)\n]\n\nOLLAMA_BASE = \"http://localhost:11434/v1\"\n\nTEMPERATURE = 0.7\nTIMEOUT_S = 600\nRETRY_ATTEMPTS = 3\nRETRY_BACKOFF_BASE_S = 5\nRETRY_BACKOFF_MAX_S = 60\n\nSYSTEM_MSG = (\n    \"You enrich a guitar-tab catalog with search metadata for a web app. \"\n    \"Output ONE JSON object only — no markdown fences, no commentary, no \"\n    \"surrounding text.\"\n)\n\n# ---------------------------------------------------------------------------\n# LLM call (mirrors evaluator's patterns: retry/backoff + think-strip)\n# ---------------------------------------------------------------------------\n\n_THINK_RE = re.compile(r\"(?:<think>)?[\\s\\S]*?</think>\\s*\", re.IGNORECASE)\n_REFLECT_RE = re.compile(r\"<reflection>[\\s\\S]*?</reflection>\\s*\", re.IGNORECASE)\n_FENCE_OPEN_RE = re.compile(r\"^```(?:json)?\\s*\", re.IGNORECASE)\n_FENCE_CLOSE_RE = re.compile(r\"\\s*```\\s*$\")\n\n\ndef strip_reasoning(text):\n    \"\"\"Remove <think>/<reflection> blocks from LLM output before JSON parse.\"\"\"\n    text = _THINK_RE.sub(\"\", text or \"\")\n    text = _REFLECT_RE.sub(\"\", text)\n    return text.strip()\n\n\ndef strip_fences(text):\n    \"\"\"Remove ```json ... ``` wrapping if the model added it despite the prompt.\"\"\"\n    text = _FENCE_OPEN_RE.sub(\"\", text)\n    text = _FENCE_CLOSE_RE.sub(\"\", text)\n    return text.strip()\n\n\ndef call_ollama(model, system_msg, user_msg, temperature=TEMPERATURE,\n                timeout=TIMEOUT_S, base_url=None, api_key=None, extra_body=None):\n    \"\"\"OpenAI-compat /chat/completions. Returns dict:\n        {content, raw, latency_ms, tokens_in, tokens_out, error}\n    On final failure (after retries), `content` is None and `error` is set.\n\n    `base_url` defaults to local Ollama; pass e.g. https://api.xiaomimimo.com/v1\n    for a hosted provider. `api_key` adds an Authorization: Bearer header\n    (local Ollama needs none; Mimo / Ollama Cloud require one). `extra_body`\n    is a dict merged into the request payload — e.g.\n    {\"thinking\": {\"type\": \"disabled\"}} to turn off MiMo's reasoning mode.\n    \"\"\"\n    base = (base_url or OLLAMA_BASE).rstrip(\"/\")\n    url = f\"{base}/chat/completions\"\n    headers = {\"Content-Type\": \"application/json\"}\n    if api_key:\n        headers[\"Authorization\"] = f\"Bearer {api_key}\"\n    payload = {\n        \"model\": model,\n        \"messages\": [\n            {\"role\": \"system\", \"content\": system_msg},\n            {\"role\": \"user\", \"content\": user_msg},\n        ],\n        \"stream\": False,\n        \"temperature\": temperature,\n    }\n    if extra_body:\n        payload.update(extra_body)\n    last_err = None\n    t0 = time.time()\n    for attempt in range(1, RETRY_ATTEMPTS + 1):\n        try:\n            resp = requests.post(url, json=payload, headers=headers, timeout=timeout)\n            if resp.status_code != 200:\n                raise RuntimeError(f\"HTTP {resp.status_code}: {resp.text[:200]}\")\n            data = resp.json()\n            latency_ms = int((time.time() - t0) * 1000)\n            content = (data.get(\"choices\") or [{}])[0].get(\"message\", {}).get(\"content\", \"\")\n            usage = data.get(\"usage\") or {}\n            return {\n                \"content\": content,\n                \"raw\": data,\n                \"latency_ms\": latency_ms,\n                \"tokens_in\": usage.get(\"prompt_tokens\"),\n                \"tokens_out\": usage.get(\"completion_tokens\"),\n                \"error\": None,\n            }\n        except Exception as err:\n            last_err = err\n            if attempt >= RETRY_ATTEMPTS:\n                break\n            delay = min(RETRY_BACKOFF_BASE_S * (2 ** (attempt - 1)),\n                        RETRY_BACKOFF_MAX_S)\n            jitter = delay * 0.1\n            wait_s = delay + (jitter * (time.time() % 1))\n            print(f\"    attempt {attempt}/{RETRY_ATTEMPTS} failed: {err} — \"\n                  f\"retrying in {wait_s:.1f}s\", file=sys.stderr)\n            time.sleep(wait_s)\n    return {\n        \"content\": None,\n        \"raw\": None,\n        \"latency_ms\": int((time.time() - t0) * 1000),\n        \"tokens_in\": None,\n        \"tokens_out\": None,\n        \"error\": str(last_err),\n    }\n\n\n# ---------------------------------------------------------------------------\n# Hallucination scoring\n# ---------------------------------------------------------------------------\n\ndef _fold(s):\n    \"\"\"Loose token-equality fold: lowercase, strip diacritics + punctuation.\"\"\"\n    s = (s or \"\").lower()\n    s = re.sub(r\"[^\\w\\s]\", \" \", s, flags=re.UNICODE)\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n    return s\n\n\ndef key_phrases_in_body(phrases, body):\n    \"\"\"Fraction of key_phrases that appear (verbatim or token-folded) in body.\n    Returns (hits, total, fraction).\"\"\"\n    if not phrases:\n        return 0, 0, 1.0  # no claim = no hallucination\n    body_folded = _fold(body)\n    hits = 0\n    for p in phrases:\n        p_folded = _fold(str(p))\n        if not p_folded:\n            continue\n        if p_folded in body_folded:\n            hits += 1\n    return hits, len(phrases), (hits / len(phrases)) if phrases else 1.0\n\n\n# ---------------------------------------------------------------------------\n# Run orchestration\n# ---------------------------------------------------------------------------\n\ndef pick_test_tabs(import_path, keys):\n    \"\"\"Find the named tabs in the UG import; return [{artist, song, body, id}].\"\"\"\n    data = json.loads(Path(import_path).read_text(encoding=\"utf-8\"))\n    tabs = data.get(\"tabs\") or []\n    picks = {}\n    for t in tabs:\n        song = t.get(\"song\") or \"\"\n        for k in keys:\n            if k in song and k not in picks:\n                picks[k] = {\n                    \"key\": k,\n                    \"artist\": t.get(\"artist\", \"\"),\n                    \"song\": song,\n                    \"body\": t.get(\"body\") or \"\",  # full body — Nemotron flat-rate makes token cost irrelevant\n                    \"id\": f\"{slugify(t.get('artist', ''))}__{slugify(song)}\",\n                }\n                break\n    missing = [k for k in keys if k not in picks]\n    if missing:\n        print(f\"WARN: didn't find these test tabs in import: {missing}\",\n              file=sys.stderr)\n    return [picks[k] for k in keys if k in picks]\n\n\ndef run_one(model, tab, base_url=None, api_key=None, extra_body=None):\n    \"\"\"Run a single (model, tab) pair. Returns the full per-run dict.\"\"\"\n    user_msg = SONG_PROMPT.format(\n        artist=tab[\"artist\"], song=tab[\"song\"], body=tab[\"body\"]\n    ) + STRICT_SUFFIX\n\n    llm = call_ollama(model, SYSTEM_MSG, user_msg, base_url=base_url,\n                      api_key=api_key, extra_body=extra_body)\n    parsed = None\n    parse_ok = False\n    parse_path = None  # \"strict\" | \"salvaged\" | \"failed\"\n\n    if llm[\"content\"]:\n        cleaned = strip_fences(strip_reasoning(llm[\"content\"]))\n        try:\n            parsed = extract_json(cleaned)\n            parse_ok = parsed is not None\n            parse_path = \"strict\" if parse_ok else None\n        except Exception:\n            parsed = None\n            parse_ok = False\n        if not parse_ok:\n            salvaged = salvage_truncated_json(cleaned)\n            if salvaged is not None:\n                parsed = salvaged\n                parse_ok = True\n                parse_path = \"salvaged\"\n        if not parse_ok:\n            parse_path = \"failed\"\n\n    key_phrases = (parsed or {}).get(\"key_phrases\") or []\n    hits, total, frac = key_phrases_in_body(key_phrases, tab[\"body\"])\n\n    return {\n        \"model\": model,\n        \"tab_key\": tab[\"key\"],\n        \"tab_id\": tab[\"id\"],\n        \"artist\": tab[\"artist\"],\n        \"song\": tab[\"song\"],\n        \"latency_ms\": llm[\"latency_ms\"],\n        \"tokens_in\": llm[\"tokens_in\"],\n        \"tokens_out\": llm[\"tokens_out\"],\n        \"error\": llm[\"error\"],\n        \"raw_content\": llm[\"content\"],\n        \"parse_ok\": parse_ok,\n        \"parse_path\": parse_path,\n        \"enrichment\": parsed,\n        \"key_phrases_hit\": hits,\n        \"key_phrases_total\": total,\n        \"key_phrases_fraction\": round(frac, 3),\n    }\n\n\ndef run_key(model, tab_id):\n    \"\"\"Filesystem-safe key for `crawler/bench/runs/<model>__<tab>.json`.\"\"\"\n    safe_model = model.replace(\":\", \"_\").replace(\"/\", \"_\")\n    return f\"{safe_model}__{tab_id}\"\n\n\n# ---------------------------------------------------------------------------\n# Compare-table renderer\n# ---------------------------------------------------------------------------\n\ndef render_compare_md(runs_dir, models, tabs):\n    \"\"\"Build COMPARE.md from existing run files.\"\"\"\n    by_model = {m: {} for m in models}\n    for f in sorted(runs_dir.glob(\"*.json\")):\n        try:\n            run = json.loads(f.read_text(encoding=\"utf-8\"))\n        except Exception:\n            continue\n        m = run[\"model\"]\n        tid = run[\"tab_id\"]\n        if m in by_model:\n            by_model[m][tid] = run\n\n    out = []\n    out.append(\"# Enrichment Bench — Ollama Cloud comparison\")\n    out.append(\"\")\n    out.append(f\"Models tested: {', '.join(f'`{m}`' for m in models)}\")\n    out.append(\"\")\n    out.append(f\"Tabs tested ({len(tabs)}):\")\n    for tab in tabs:\n        out.append(f\"- **{tab['key']}** — {tab['artist']} — {tab['song']}  \"\n                   f\"(id `{tab['id']}`)\")\n    out.append(\"\")\n\n    # Per-tab tables\n    out.append(\"## Per-tab results\")\n    out.append(\"\")\n    for tab in tabs:\n        out.append(f\"### {tab['key']} — {tab['artist']}\")\n        out.append(\"\")\n        out.append(\"| Model | parse | latency | tokens (in/out) | kp hit/total | \"\n                   \"language | themes | key_phrases (first 2) |\")\n        out.append(\"|---|---|---|---|---|---|---|---|\")\n        for m in models:\n            run = by_model.get(m, {}).get(tab[\"id\"])\n            if not run:\n                out.append(f\"| `{m}` | — | — | — | — | — | — | _(no run)_ |\")\n                continue\n            parse = \"✓\" if run[\"parse_ok\"] else \"✗\"\n            if run[\"parse_path\"] == \"salvaged\":\n                parse = \"✓ (salvaged)\"\n            lat = f\"{run['latency_ms']/1000:.1f}s\"\n            ti, to = run[\"tokens_in\"], run[\"tokens_out\"]\n            toks = f\"{ti}/{to}\" if ti is not None and to is not None else \"—\"\n            kp = f\"{run['key_phrases_hit']}/{run['key_phrases_total']}\"\n            enrich = run[\"enrichment\"] or {}\n            lang = enrich.get(\"language\", \"—\")\n            themes = \", \".join((enrich.get(\"themes\") or [])[:3]) or \"—\"\n            phrases = (enrich.get(\"key_phrases\") or [])[:2]\n            phrases_str = \" / \".join(f\"_{p}_\" for p in phrases) or \"—\"\n            err_note = f\" ⚠ {run['error']}\" if run[\"error\"] else \"\"\n            out.append(f\"| `{m}` | {parse}{err_note} | {lat} | {toks} | \"\n                       f\"{kp} | {lang} | {themes} | {phrases_str} |\")\n        out.append(\"\")\n\n    # Aggregate stats\n    out.append(\"## Aggregate (across all tabs)\")\n    out.append(\"\")\n    out.append(\"| Model | parse rate | mean latency | mean tokens out | \"\n               \"mean kp hit-rate |\")\n    out.append(\"|---|---|---|---|---|\")\n    for m in models:\n        runs = list(by_model.get(m, {}).values())\n        if not runs:\n            out.append(f\"| `{m}` | — | — | — | — |\")\n            continue\n        n = len(runs)\n        parse_n = sum(1 for r in runs if r[\"parse_ok\"])\n        lat_avg_s = sum(r[\"latency_ms\"] for r in runs) / n / 1000\n        toks_out = [r[\"tokens_out\"] for r in runs if r[\"tokens_out\"]]\n        toks_avg = (sum(toks_out) / len(toks_out)) if toks_out else None\n        kp_fracs = [r[\"key_phrases_fraction\"] for r in runs if r[\"parse_ok\"]]\n        kp_avg = (sum(kp_fracs) / len(kp_fracs)) if kp_fracs else None\n\n        parse_cell = f\"{parse_n}/{n} ({100*parse_n/n:.0f}%)\"\n        lat_cell = f\"{lat_avg_s:.1f}s\"\n        toks_cell = f\"{toks_avg:.0f}\" if toks_avg is not None else \"—\"\n        kp_cell = f\"{kp_avg*100:.0f}%\" if kp_avg is not None else \"—\"\n        out.append(f\"| `{m}` | {parse_cell} | {lat_cell} | {toks_cell} | {kp_cell} |\")\n    out.append(\"\")\n    out.append(\"**Read this table as**: parse-rate is the \"\n               \"first quality gate (must be ~100%); key_phrases hit-rate is \"\n               \"the hallucination signal (lower = more invented quotes); \"\n               \"latency matters most for batch enrichment of large UG imports.\")\n    return \"\\n\".join(out)\n\n\n# ---------------------------------------------------------------------------\n# Main\n# ---------------------------------------------------------------------------\n\ndef parse_args():\n    p = argparse.ArgumentParser()\n    p.add_argument(\"--input\", default=\"crawler/private/ug-import.json\")\n    p.add_argument(\"--out-dir\", default=\"crawler/bench\")\n    p.add_argument(\"--models\", nargs=\"*\", default=DEFAULT_MODELS)\n    p.add_argument(\"--base\", default=OLLAMA_BASE,\n                   help=f\"OpenAI-compat base URL (default: {OLLAMA_BASE}). \"\n                        \"Use https://api.mimo.xiaomi.com/v1 for Mimo, \"\n                        \"https://ollama.com/v1 for Ollama Cloud direct, etc.\")\n    p.add_argument(\"--api-key-env\", default=None,\n                   help=\"Name of an env var holding the Bearer API key. \"\n                        \"Omit for local Ollama (no auth). E.g. MIMO_API_KEY.\")\n    p.add_argument(\"--extra-body\", default=None,\n                   help=\"JSON string merged into the request payload. \"\n                        \"E.g. '{\\\"thinking\\\": {\\\"type\\\": \\\"disabled\\\"}}' to \"\n                        \"turn off MiMo's reasoning mode.\")\n    p.add_argument(\"--force\", action=\"store_true\",\n                   help=\"Re-run runs even if a cached file exists\")\n    p.add_argument(\"--compare-only\", action=\"store_true\",\n                   help=\"Skip LLM calls, just regenerate COMPARE.md\")\n    return p.parse_args()\n\n\ndef main():\n    reconfigure_streams()\n    args = parse_args()\n    out_dir = Path(args.out_dir)\n    runs_dir = out_dir / \"runs\"\n    runs_dir.mkdir(parents=True, exist_ok=True)\n\n    # Resolve the API key from the named env var, if any. Never logged.\n    import os\n    api_key = os.environ.get(args.api_key_env) if args.api_key_env else None\n    if args.api_key_env and not api_key:\n        print(f\"ABORT: --api-key-env {args.api_key_env} is set but the env \"\n              f\"var is empty/unset.\", file=sys.stderr)\n        return 1\n\n    extra_body = None\n    if args.extra_body:\n        try:\n            extra_body = json.loads(args.extra_body)\n        except json.JSONDecodeError as e:\n            print(f\"ABORT: --extra-body is not valid JSON: {e}\", file=sys.stderr)\n            return 1\n\n    tabs = pick_test_tabs(args.input, TEST_TAB_KEYS)\n    if not tabs:\n        print(f\"ABORT: no test tabs found in {args.input}\", file=sys.stderr)\n        return 1\n\n    # Persist the inputs so a reader can see exactly what was tested\n    (out_dir / \"inputs.json\").write_text(\n        json.dumps(tabs, indent=2, ensure_ascii=False), encoding=\"utf-8\"\n    )\n\n    if args.compare_only:\n        md = render_compare_md(runs_dir, args.models, tabs)\n        (out_dir / \"COMPARE.md\").write_text(md, encoding=\"utf-8\")\n        print(f\"Wrote {out_dir / 'COMPARE.md'} (compare-only).\")\n        return 0\n\n    total = len(args.models) * len(tabs)\n    done = 0\n    auth_note = f\" (auth via ${args.api_key_env})\" if args.api_key_env else \"\"\n    print(f\"Running {total} (model × tab) combinations against {args.base}{auth_note}\")\n    for model in args.models:\n        for tab in tabs:\n            done += 1\n            key = run_key(model, tab[\"id\"])\n            run_path = runs_dir / f\"{key}.json\"\n            if run_path.exists() and not args.force:\n                print(f\"[{done:2}/{total}] cached: {model} × {tab['key']}\")\n                continue\n            print(f\"[{done:2}/{total}] {model} × {tab['key']} ...\", flush=True)\n            result = run_one(model, tab, base_url=args.base, api_key=api_key,\n                             extra_body=extra_body)\n            run_path.write_text(\n                json.dumps(result, indent=2, ensure_ascii=False),\n                encoding=\"utf-8\"\n            )\n            status = \"OK\" if result[\"parse_ok\"] else \"FAIL\"\n            if result[\"error\"]:\n                status = f\"ERROR ({result['error'][:60]})\"\n            kp = f\"{result['key_phrases_hit']}/{result['key_phrases_total']}\"\n            print(f\"    → {status} | {result['latency_ms']/1000:.1f}s | \"\n                  f\"kp {kp}\")\n\n    md = render_compare_md(runs_dir, args.models, tabs)\n    (out_dir / \"COMPARE.md\").write_text(md, encoding=\"utf-8\")\n    print(f\"\\nDone. See {out_dir / 'COMPARE.md'}.\")\n    return 0\n\n\nif __name__ == \"__main__\":\n    sys.exit(main())\n"}}}}