hack/tools/scripts/llm-key-hunter/results/_cache/content/avitrost_harness-complexity.json

{"repo": "avitrost/harness-complexity", "files": {"viz/make_sparkline_tables_v2.py": {"c6dbe343d3b9c93910b2c532f7dd62083f078ee7": {"t": 1786068565.6336076, "n": 34536, "h": "13c648ef4c8c6b33", "c": "#!/usr/bin/env python3\n\"\"\"Generate Tufte-style sparkline tables from the merged provider-matrix aggregate.\n\nv2 adds a third deliverable: the gpt-5.5/high *retry ablation* for the two\nmini-swe-agent variants (mini-swe-agent and msa-prompt-compressed) across retry limits\n0..3. It also keeps the original two deliverables unchanged in style:\n\n  1. One table per reasoning effort (rows = harnesses by LOC, cols = model).\n  2. The transpose: one table per model (rows = harnesses, cols = effort).\n  3. Retry ablation: rows = {mini-swe-agent, msa-prompt-compressed}, cols = retry 0..3.\n\nCI in every cell is the Wilson binomial interval over the 90 trials (per-trial\nsampling noise on these exact 9 tasks), matching the original deliverable.\n\nTo drop Anthropic / TBLite gpt-5.5 in later: add their (model, display) entries\nto FAMILIES, append their aggregate path to AGG_SOURCES, and rerun. Cells with\nno data render as a dash + broken line; nothing is fabricated.\n\"\"\"\nimport csv\nimport math\nimport os\n\nHERE = os.path.dirname(os.path.abspath(__file__))\n# Data source is absolute so the script keeps working wherever it's moved;\n# the HTML is written next to the script (HERE).\nAGG_DIR = (\"/wbl-fast/usrs/trost/harness-complexity/final_test/\"\n           \"aggregate_openai_deepseek_20260609\")\nSRC = os.path.join(AGG_DIR, \"aggregate_by_candidate_with_prior_openai10.csv\")\n# Anthropic \"best complete run\" manual aggregate (same schema: candidate,\n# effort, model, num_successes, num_crashes, num_attempts=90 per cell). We\n# recompute Wilson-over-90 from successes like every other provider, so the\n# CSV's normal-approx ci columns are not used.\nANTH_SRC = (\"/wbl-fast/usrs/trost/harness-complexity/final_test/\"\n            \"manual_aggregate_anthropic_best_20260612_0028/aggregate_by_candidate.csv\")\nCANDIDATE_SOURCES = [SRC, ANTH_SRC]\nOUT = os.path.join(HERE, \"sparkline_tables_v2.html\")\n\n# Retry-ablation sources (gpt-5.5 / high, same 9 tasks).\nRETRY_DIR = (\"/wbl-fast/usrs/trost/harness-complexity/final_test/\"\n             \"mini_swe_retry_matrix_gpt55_high_20260609_204939\")\nRETRY_SCORES = os.path.join(RETRY_DIR, \"scores_by_harness.csv\")\n\nEXPECTED_TRIALS = 90  # 9 tasks x 10 trials\n\n# ---- fixed scale (shared by the main sparklines) ----\nSCALE_LO, SCALE_HI = 10.0, 70.0\n# the retry ablation lives in a tight 59-77% band -> its own scale so the\n# trend across retries is legible instead of clipped at the global ceiling.\nRETRY_LO, RETRY_HI = 50.0, 80.0\n\n# ---- harness order: ascending by LOC ----\n# col key -> (display name, LOC)\nHARNESS = [\n    (\"minimal\",    (\"minimal-agent\",         100)),\n    (\"mini_bare\",  (\"msa-fully-compressed\",          149)),\n    (\"c400\",       (\"codex-400\",             398)),\n    (\"bare_v2\",    (\"msa-prompt-compressed\",          408)),\n    (\"mini_v2\",    (\"mini-swe-agent\",        478)),\n    (\"term2_comp\", (\"terminus-2-compressed\", 634)),\n    (\"c700\",       (\"codex-700\",             700)),\n    (\"c1000\",      (\"codex-1000\",           1000)),\n    (\"c1300\",      (\"codex-1300\",           1300)),\n    (\"c_comp\",     (\"codex-compressed\",     1660)),\n    (\"c_full\",     (\"codex-full\",           2210)),\n]\nHARNESS_ORDER = [k for k, _ in HARNESS]\nHARNESS_META = dict(HARNESS)\n\nCANDIDATE_TO_KEY = {\n    \"seed_codex_1000\": \"c1000\",\n    \"seed_codex_1300\": \"c1300\",\n    \"seed_codex_400\": \"c400\",\n    \"seed_codex_700\": \"c700\",\n    \"seed_codex_compressed\": \"c_comp\",\n    \"seed_codex_full\": \"c_full\",\n    \"seed_mini_swe_agent_barebones\": \"mini_bare\",      # v1\n    \"seed_mini_swe_agent_barebones_v2\": \"bare_v2\",     # v2\n    \"seed_mini_swe_agent_v2\": \"mini_v2\",\n    \"seed_minimal_agent\": \"minimal\",\n    \"seed_terminus_2_compressed\": \"term2_comp\",\n}\n\n# ---- model families, small -> large ----\n# family -> list of (model id, display name)\nFAMILIES = [\n    (\"OpenAI\", [\n        (\"gpt-5.4-mini\", \"gpt-5.4-mini\"),\n        (\"gpt-5.4\",      \"gpt-5.4\"),\n        (\"gpt-5.5\",      \"gpt-5.5\"),\n    ]),\n    (\"Anthropic\", [\n        (\"claude-haiku-4-5\",  \"Haiku 4.5\"),\n        (\"claude-sonnet-4-6\", \"Sonnet 4.6\"),\n        (\"claude-opus-4-8\",   \"Opus 4.8\"),\n    ]),\n    (\"DeepSeek\", [\n        (\"deepseek-v4-flash\", \"v4-flash\"),\n        (\"deepseek-v4-pro\",   \"v4-pro\"),\n    ]),\n]\nMODEL_DISPLAY = {m: d for _, ms in FAMILIES for m, d in ms}\nMODEL_FAMILY = {m: fam for fam, ms in FAMILIES for m, _ in ms}\nMODEL_ORDER = [m for _, ms in FAMILIES for m, _ in ms]\n\n# effort order per family (least -> most reasoning). Anthropic: haiku ran at\n# \"none\" only; sonnet/opus at low/medium/high. Absent efforts are filtered out.\nEFFORT_ORDER = {\n    \"OpenAI\": [\"low\", \"medium\", \"high\"],\n    \"DeepSeek\": [\"none\", \"high\", \"max\"],\n    \"Anthropic\": [\"none\", \"low\", \"medium\", \"high\"],\n}\n\nZ = 1.96  # 95%\n\n\ndef wilson_half(succ, n):\n    \"\"\"Half-width (in pp) of the Wilson score interval for succ/n.\n\n    Trial-level binomial CI on these exact tasks (per-trial sampling noise),\n    not the task-level CI shipped in the source file. Wilson is used instead of\n    Wald so cells at 0 or n successes still get a sensible, non-zero interval.\n    \"\"\"\n    if n == 0:\n        return None\n    p = succ / n\n    denom = 1.0 + Z * Z / n\n    margin = (Z / denom) * math.sqrt(p * (1 - p) / n + Z * Z / (4.0 * n * n))\n    return margin * 100.0\n\n\n# ---------------------------------------------------------------- load\ndef load():\n    \"\"\"Return data[(model, effort, harness_key)] = cell dict, or absent if missing.\n\n    Reads every candidate source (OpenAI+DeepSeek merge, then Anthropic). All use\n    90 attempts/cell and the same columns; msa-prompt-compressed (Anthropic-only here) is\n    dropped by CANDIDATE_TO_KEY so the proxy tables keep the canonical 10 rows.\n    \"\"\"\n    data = {}\n    for src in CANDIDATE_SOURCES:\n        with open(src) as f:\n            for r in csv.DictReader(f):\n                hk = CANDIDATE_TO_KEY.get(r[\"candidate\"])\n                if hk is None:\n                    continue\n                succ = int(r[\"num_successes\"])\n                score = 100.0 * succ / EXPECTED_TRIALS  # crashes count as failures\n                data[(r[\"model\"], r[\"effort\"], hk)] = {\n                    \"score\": score,\n                    \"half\": wilson_half(succ, EXPECTED_TRIALS),\n                    \"succ\": succ,\n                    \"crashes\": int(r[\"num_crashes\"]),\n                    \"attempts\": int(r[\"num_attempts\"]),\n                }\n    return data\n\n\n# ---- retry ablation: rows = harness variant, cols = retry limit 0..3 ----\n# display label -> retry-sweep `harness` prefix (cells named <prefix>_r<N>)\nRETRY_ROWS = [\n    (\"mini-swe-agent\", \"mini_v2\"),\n    (\"msa-prompt-compressed\",   \"barebones_v2\"),\n]\nRETRY_LIMITS = [0, 1, 2, 3]\n\n\ndef load_retry():\n    \"\"\"retry[(label, retry)] = cell dict.\n\n    Cells come from the dedicated retry sweep. The mini-swe-agent r3 cell is\n    absent from that sweep -- its r3 (=default 3-retry run) is the provider\n    matrix datapoint already in SRC (gpt-5.5/high/seed_mini_swe_agent_v2).\n    \"\"\"\n    by_harness = {}\n    with open(RETRY_SCORES) as f:\n        for r in csv.DictReader(f):\n            succ = int(r[\"successes\"])\n            n = int(r[\"N\"])\n            by_harness[r[\"harness\"]] = {\n                \"score\": 100.0 * succ / n,\n                \"half\": wilson_half(succ, n),\n                \"succ\": succ,\n                \"attempts\": n,\n            }\n\n    # mini-swe-agent r3 default, pulled from the main aggregate.\n    default_r3 = None\n    with open(SRC) as f:\n        for r in csv.DictReader(f):\n            if (r[\"model\"], r[\"effort\"], r[\"candidate\"]) == (\n                    \"gpt-5.5\", \"high\", \"seed_mini_swe_agent_v2\"):\n                succ = int(r[\"num_successes\"])\n                default_r3 = {\n                    \"score\": 100.0 * succ / EXPECTED_TRIALS,\n                    \"half\": wilson_half(succ, EXPECTED_TRIALS),\n                    \"succ\": succ,\n                    \"attempts\": int(r[\"num_attempts\"]),\n                    \"is_default\": True,\n                }\n\n    retry = {}\n    for label, prefix in RETRY_ROWS:\n        for n in RETRY_LIMITS:\n            cell = by_harness.get(f\"{prefix}_r{n}\")\n            if cell is None and label == \"mini-swe-agent\" and n == 3:\n                cell = default_r3\n            if cell is not None:\n                retry[(label, n)] = cell\n    return retry\n\n\n# ---- TBLite: a separate (broader) 100-task benchmark, 1 trial/task. Several runs\n# across model/effort configs, shown as their own standalone tables on the right.\nTBLITE_BASE = \"/wbl-fast/usrs/trost/harness-complexity/final_test\"\n# (model, effort, run dir). The model/effort is not in the records -> taken here.\nTBLITE_ROOTS = [\n    (\"gpt-5.5\", \"high\", \"tblite_gpt55_high_1x100_c200_20260611_190038\"),\n    (\"gpt-5.5\", \"high\", \"tblite_gpt55_high_barebones_v1_c100_20260611_220336\"),\n    (\"gpt-5.4\", \"low\",  \"tblite_gpt54_low_barebonesv2_vs_codexfull_c200_20260611_221815\"),\n    (\"gpt-5.4-mini\", \"low\",\n     \"tblite_gpt54mini_low_barebonesv2_vs_codexfull_c200_20260611_221815\"),\n]\n# TBLite seed dir -> row key. barebones v1 (149 LOC) and v2 (408) are distinct.\nTBLITE_SEED_TO_KEY = {\n    \"seed_codex_1000\": \"c1000\", \"seed_codex_1300\": \"c1300\",\n    \"seed_codex_400\": \"c400\", \"seed_codex_700\": \"c700\",\n    \"seed_codex_compressed\": \"c_comp\", \"seed_codex_full\": \"c_full\",\n    \"seed_mini_swe_agent_v2\": \"mini_v2\", \"seed_minimal_agent\": \"minimal\",\n    \"seed_terminus_2_compressed\": \"term2_comp\",\n    \"seed_mini_swe_agent_barebones\": \"bare_v1\",\n    \"seed_mini_swe_agent_barebones_v2\": \"bare_v2\",\n}\n# TBLite's own harness order/labels (ascending LOC). Both barebones variants shown.\nTBLITE_HARNESS = [\n    (\"minimal\",    (\"minimal-agent\",         100)),\n    (\"bare_v1\",    (\"msa-fully-compressed\",          149)),\n    (\"c400\",       (\"codex-400\",             398)),\n    (\"bare_v2\",    (\"msa-prompt-compressed\",          408)),\n    (\"mini_v2\",    (\"mini-swe-agent\",        478)),\n    (\"term2_comp\", (\"terminus-2-compressed\", 634)),\n    (\"c700\",       (\"codex-700\",             700)),\n    (\"c1000\",      (\"codex-1000\",           1000)),\n    (\"c1300\",      (\"codex-1300\",           1300)),\n    (\"c_comp\",     (\"codex-compressed\",     1660)),\n    (\"c_full\",     (\"codex-full\",           2210)),\n]\n# msa-prompt-compressed vs codex-full across model strength (ascending capability).\nTBLITE_CMP_ROWS = [(\"bare_v2\", \"msa-prompt-compressed\"), (\"c_full\", \"codex-full\")]\nTBLITE_CMP_COLS = [\n    (\"gpt-5.4-mini\", \"low\",  \"5.4-mini<br>low\"),\n    (\"gpt-5.4\",      \"low\",  \"5.4<br>low\"),\n    (\"gpt-5.5\",      \"high\", \"5.5<br>high\"),\n]\n\n\ndef load_tblite():\n    \"\"\"tblite[(model, effort, harness_key)] = cell (Wilson CI over the 100 trials).\"\"\"\n    import json\n    import os\n    import re\n    tbl = {}\n    for model, effort, root in TBLITE_ROOTS:\n        path = os.path.join(TBLITE_BASE, root, \"summary.json\")\n        with open(path) as f:\n            recs = json.load(f)\n        for e in recs:\n            cmd = \" \".join(e[\"command\"]) if isinstance(e.get(\"command\"), list) else \"\"\n            m = re.search(r\"seed_[a-z0-9_]+\", cmd)  # captures the full _v2 suffix\n            hk = TBLITE_SEED_TO_KEY.get(m.group(0)) if m else None\n            if hk is None:\n                continue\n            n = int(e[\"num_trials\"])\n            succ = int(e[\"num_successes\"])  # crashes excluded -> counted as fails\n            tbl[(model, effort, hk)] = {\n                \"score\": 100.0 * succ / n,\n                \"half\": wilson_half(succ, n),\n                \"succ\": succ,\n                \"attempts\": n,\n            }\n    return tbl\n\n\n# ---------------------------------------------------------------- geometry\ndef y_of(score, top, plot_h, lo=SCALE_LO, hi=SCALE_HI):\n    \"\"\"Map a score% to a y coord on a [lo, hi] scale, clamped to that band.\"\"\"\n    s = max(lo, min(hi, score))\n    frac = (s - lo) / (hi - lo)\n    return top + (1.0 - frac) * plot_h\n\n\ndef spark_svg(scores, *, w, h, dot_color, heavy=False, pad=2.5, dot_r=2.4,\n              lo=SCALE_LO, hi=SCALE_HI):\n    \"\"\"Build an SVG sparkline. `scores` is a list of float|None across columns.\n\n    Missing points are skipped entirely: the line connects straight through the\n    present points (no break, no dash), as if the missing column were not there.\n    \"\"\"\n    n = len(scores)\n    top, plot_h = pad, h - 2 * pad\n    if n == 1:\n        xs = [w / 2.0]\n    else:\n        left, right = pad + 1.0, w - pad - 1.0\n        xs = [left + i * (right - left) / (n - 1) for i in range(n)]\n\n    # present points only, kept in column order; gaps are simply dropped\n    pts = [(xs[i], y_of(scores[i], top, plot_h, lo, hi))\n           for i in range(n) if scores[i] is not None]\n\n    sw = \"1.5\" if heavy else \"1.1\"\n    parts = []\n    if len(pts) == 1:\n        x, y = pts[0]\n        parts.append(f'<circle cx=\"{x:.1f}\" cy=\"{y:.1f}\" r=\"1.2\" '\n                     f'fill=\"var(--ink)\" class=\"spark-pt\"/>')\n    elif len(pts) >= 2:\n        d = \"M \" + \" L \".join(f\"{x:.1f} {y:.1f}\" for x, y in pts)\n        parts.append(\n            f'<path d=\"{d}\" fill=\"none\" stroke=\"var(--line)\" '\n            f'stroke-width=\"{sw}\" stroke-linejoin=\"round\" stroke-linecap=\"round\"/>'\n        )\n\n    # endpoint dot on the last present point (largest model / heaviest harness)\n    if pts:\n        x, y = pts[-1]\n        parts.append(f'<circle cx=\"{x:.1f}\" cy=\"{y:.1f}\" r=\"{dot_r}\" fill=\"{dot_color}\"/>')\n\n    cls = \"spark tall\" if heavy else \"spark\"\n    return (f'<svg class=\"{cls}\" viewBox=\"0 0 {w:.0f} {h:.0f}\" '\n            f'preserveAspectRatio=\"none\" role=\"img\">{\"\".join(parts)}</svg>')\n\n\ndef cell_text(cell):\n    if cell is None:\n        return '<span class=\"miss\">—</span>'\n    s = f\"{round(cell['score'])}\"\n    if cell[\"half\"] is not None:\n        s = f'{s}<span class=\"ci\"> ±{round(cell[\"half\"])}</span>'\n    if cell.get(\"flag\"):\n        s += f'<span class=\"ci\"> {cell[\"flag\"]}</span>'\n    return s\n\n\n# ---------------------------------------------------------------- table builders\nROW_W, ROW_H = 88, 18      # row sparkline (\"vs ...\")\nFOOT_W, FOOT_H = 88, 90    # footer sparkline (\"vs harness\")\nCORAL = \"#D85A30\"\n\n\ndef build_table(*, columns, col_family, scores_for, caption, vs_label,\n                col_header_fn, family_groups=None, highlight=None):\n    \"\"\"columns: list of column keys. scores_for(harness_key, col) -> cell|None.\"\"\"\n    out = ['<table class=\"grid\">']\n\n    # optional family-group header row\n    if family_groups:\n        out.append(\"<thead><tr><th class=\\\"corner\\\"></th>\")\n        for fam, span in family_groups:\n            out.append(f'<th class=\"fam\" colspan=\"{span}\">{fam}</th>')\n        out.append('<th class=\"corner\"></th></tr>')\n        hdr_open = \"<tr>\"\n    else:\n        out.append(\"<thead>\")\n        hdr_open = \"<tr>\"\n\n    # which columns begin a new family (=> draw a stronger vertical divider)\n    fam_start = []\n    prev_fam = None\n    for c in columns:\n        fam = col_family.get(c)\n        fam_start.append(prev_fam is not None and fam != prev_fam)\n        prev_fam = fam\n\n    def famcls(base, i):\n        return f\"{base} famstart\" if fam_start[i] else base\n\n    # column header row\n    out.append(hdr_open)\n    out.append('<th class=\"rowlab\">harness <span class=\"ci\">loc</span></th>')\n    for i, c in enumerate(columns):\n        out.append(f'<th class=\"{famcls(\"colh\", i)}\">{col_header_fn(c)}</th>')\n    out.append(f'<th class=\"sparkh\">{vs_label}</th></tr></thead><tbody>')\n\n    # data rows\n    for hk in HARNESS_ORDER:\n        name, loc = HARNESS_META[hk]\n        is_hi = (highlight == hk)\n        row_scores = [scores_for(hk, c) for c in columns]\n        lab_cls = \"rowlab hl\" if is_hi else \"rowlab\"\n        out.append('<tr class=\"minrow\">' if hk == \"minimal\" else \"<tr>\")\n        out.append(\n            f'<td class=\"{lab_cls}\">{name} <span class=\"ci\">{loc}</span></td>'\n        )\n        for i, cell in enumerate(row_scores):\n            out.append(f'<td class=\"{famcls(\"num\", i)}\">{cell_text(cell)}</td>')\n        spk = spark_svg([c[\"score\"] if c else None for c in row_scores],\n                        w=ROW_W, h=ROW_H, dot_color=CORAL, heavy=is_hi)\n        out.append(f'<td class=\"sp\">{spk}</td>')\n        out.append(\"</tr>\")\n\n    # footer: one tall sparkline per column, across harnesses in LOC order.\n    # minimal-agent is excluded -- it's a low outlier that distorts the profile.\n    foot_harnesses = [hk for hk in HARNESS_ORDER if hk != \"minimal\"]\n    out.append('</tbody><tfoot><tr>')\n    out.append('<td class=\"rowlab foot\">vs harness ↓ <span class=\"ci\">loc→</span>'\n               '<br><span class=\"ci\">ex. minimal</span></td>')\n    for i, c in enumerate(columns):\n        col_scores = []\n        for hk in foot_harnesses:\n            cell = scores_for(hk, c)\n            col_scores.append(cell[\"score\"] if cell else None)\n        spk = spark_svg(col_scores, w=FOOT_W, h=FOOT_H,\n                        dot_color=\"var(--ink)\", heavy=True, dot_r=3.0)\n        out.append(f'<td class=\"{famcls(\"sp foot\", i)}\">{spk}</td>')\n    out.append('<td class=\"sp foot blank\"></td></tr></tfoot></table>')\n\n    return (f'<figure><figcaption>{caption}</figcaption>'\n            + \"\".join(out) + \"</figure>\")\n\n\ndef build_retry_table(retry, caption):\n    \"\"\"Retry ablation: rows = harness variant, cols = retry limit 0..3.\n\n    Own scale (RETRY_LO..RETRY_HI). Row sparkline traces score vs retry count;\n    the coral dot marks the highest retry budget. A dashed trailing segment +\n    a default-cell marker flag the mini-swe-agent r3 = default-run cell.\n    \"\"\"\n    out = ['<table class=\"grid\">']\n    out.append('<thead><tr>')\n    out.append('<th class=\"rowlab\">harness</th>')\n    for n in RETRY_LIMITS:\n        out.append(f'<th class=\"colh\">r{n}</th>')\n    out.append('<th class=\"sparkh\">vs retries →</th></tr></thead><tbody>')\n\n    for label, _prefix in RETRY_ROWS:\n        cells = [retry.get((label, n)) for n in RETRY_LIMITS]\n        out.append('<tr>')\n        out.append(f'<td class=\"rowlab\">{label}</td>')\n        for n, cell in zip(RETRY_LIMITS, cells):\n            txt = cell_text(cell)\n            if cell is not None and cell.get(\"is_default\"):\n                txt += '<span class=\"ci\"> †</span>'\n            out.append(f'<td class=\"num\">{txt}</td>')\n        spk = spark_svg([c[\"score\"] if c else None for c in cells],\n                        w=ROW_W, h=ROW_H, dot_color=CORAL,\n                        lo=RETRY_LO, hi=RETRY_HI)\n        out.append(f'<td class=\"sp\">{spk}</td>')\n        out.append('</tr>')\n\n    out.append('</tbody></table>')\n    return (f'<figure><figcaption>{caption}</figcaption>'\n            + \"\".join(out) + \"</figure>\")\n\n\ndef build_tblite_profile(tblite):\n    \"\"\"TBLite gpt-5.5/high harness profile: rows = harness (LOC ↑), one pass col.\n\n    A separate 100-task benchmark (1 trial/task) shown apart from the 9-task\n    proxy. A tall footer sparkline traces the harness profile. Both barebones\n    variants are present (v1 @149, v2 @408).\n    \"\"\"\n    cap = ('<b>TBLite · gpt-5.5 · high</b> · 100-task set, 1 trial/task · '\n           'pass-rate % with 95% Wilson CI over 100 · rows by LOC ↑ '\n           '<span class=\"note\">· a broader benchmark than the 9-task proxy at '\n           'left: compare the harness <i>profile</i>, not identical tasks.</span>')\n    out = ['<table class=\"grid\">']\n    out.append('<thead><tr><th class=\"rowlab\">harness <span class=\"ci\">loc</span></th>'\n               '<th class=\"colh\">pass %</th></tr></thead><tbody>')\n    for hk, (name, loc) in TBLITE_HARNESS:\n        out.append('<tr class=\"minrow\">' if hk == \"minimal\" else '<tr>')\n        out.append(f'<td class=\"rowlab\">{name} <span class=\"ci\">{loc}</span></td>')\n        out.append(f'<td class=\"num\">{cell_text(tblite.get((\"gpt-5.5\", \"high\", hk)))}</td>')\n        out.append('</tr>')\n    col_scores = [(tblite.get((\"gpt-5.5\", \"high\", hk)) or {}).get(\"score\")\n                  for hk, _ in TBLITE_HARNESS if hk != \"minimal\"]\n    spk = spark_svg(col_scores, w=FOOT_W, h=FOOT_H, dot_color=\"var(--ink)\",\n                    heavy=True, dot_r=3.0)\n    out.append('</tbody><tfoot><tr><td class=\"rowlab foot\">vs harness ↓ '\n               '<span class=\"ci\">loc→ · ex. minimal</span></td>'\n               f'<td class=\"sp foot\">{spk}</td></tr></tfoot>')\n    out.append('</table>')\n    return (f'<figure><figcaption>{cap}</figcaption>'\n            + \"\".join(out) + \"</figure>\")\n\n\ndef build_tblite_compare(tblite):\n    \"\"\"TBLite: msa-prompt-compressed vs codex-full across model strength (weak→strong).\n\n    Row sparkline traces each harness as the model gets stronger; the gap\n    between the heavy and minimal harness is the story.\n    \"\"\"\n    cap = ('<b>TBLite · msa-prompt-compressed vs codex-full</b> · pass-rate % with 95% '\n           'Wilson CI over 100 · cols = model, weak→strong '\n           '<span class=\"note\">· how much the heavy harness helps as the model '\n           'strengthens. Sparkline scale matches the profile above.</span>')\n    out = ['<table class=\"grid cmp\">']\n    out.append('<thead><tr><th class=\"rowlab\">harness</th>')\n    for _m, _e, lab in TBLITE_CMP_COLS:\n        out.append(f'<th class=\"colh\">{lab}</th>')\n    out.append('<th class=\"sparkh\">vs model →</th></tr></thead><tbody>')\n    for hk, name in TBLITE_CMP_ROWS:\n        cells = [tblite.get((m, e, hk)) for m, e, _ in TBLITE_CMP_COLS]\n        out.append('<tr>')\n        out.append(f'<td class=\"rowlab\">{name}</td>')\n        for cell in cells:\n            out.append(f'<td class=\"num\">{cell_text(cell)}</td>')\n        spk = spark_svg([c[\"score\"] if c else None for c in cells],\n                        w=ROW_W, h=ROW_H, dot_color=CORAL)\n        out.append(f'<td class=\"sp\">{spk}</td>')\n        out.append('</tr>')\n    out.append('</tbody></table>')\n    return (f'<figure><figcaption>{cap}</figcaption>'\n            + \"\".join(out) + \"</figure>\")\n\n\ndef build_harness_diff():\n    \"\"\"Quick prose summary of how the three msa-lineage harnesses differ.\"\"\"\n    return (\n        '<p class=\"hdiff\"><b>msa-prompt-compressed</b> and <b>mini-swe-agent</b> '\n        'are the <i>same harness mechanically</i> — same loop, assistant/tool-call '\n        'replay, format retries, env prefixing and logging. They differ only in the '\n        'instance prompt: mini-swe-agent ships the full prompt (workflow guidance, '\n        'reasoning-text instruction, an example, system info, command snippets); '\n        'msa-prompt-compressed strips that down to command rules + submit format.</p>'\n        '<p class=\"hdiff\"><b>msa-fully-compressed</b> (the 149-line harness) is a '\n        'different, much simpler agent — not just a shorter prompt. It flattens the '\n        'whole history into one user message each turn (no assistant/tool-call '\n        'replay), has no format retries (a malformed or no-tool response ends the '\n        'task instead of being corrected), uses plain-text observations, and skips '\n        'the env prefixing and rich metadata. It is closer to a minimal for-loop '\n        'agent.</p>')\n\n\n# ---------------------------------------------------------------- assemble\ndef main():\n    data = load()\n    retry = load_retry()\n    tblite = load_tblite()\n\n    # gpt msa-prompt-compressed was never run in the TB2 provider matrix; the only TB2\n    # datapoint we have is gpt-5.5/high from the retry sweep r3 (= default\n    # 3-retry config). Inject it (flagged ‡) so the msa-prompt-compressed row shows it.\n    r3 = retry.get((\"msa-prompt-compressed\", 3))\n    if r3 and (\"gpt-5.5\", \"high\", \"bare_v2\") not in data:\n        data[(\"gpt-5.5\", \"high\", \"bare_v2\")] = {\n            \"score\": r3[\"score\"], \"half\": r3[\"half\"], \"succ\": r3[\"succ\"],\n            \"crashes\": 0, \"attempts\": r3[\"attempts\"], \"flag\": \"‡\",\n        }\n\n    # ---- Deliverable 1: one table per (family, effort) ----\n    # Never mix providers in one table: OpenAI \"reasoning = high\" and DeepSeek\n    # \"thinking = high\" are different knobs, so each family gets its own tables.\n    EFFORT_WORD = {\"OpenAI\": \"reasoning\", \"DeepSeek\": \"thinking\",\n                   \"Anthropic\": \"thinking\"}\n    sec1 = []\n    for fam, fam_models in FAMILIES:\n        fam_ids = [m for m, _ in fam_models]\n        for eff in EFFORT_ORDER[fam]:\n            models = [m for m in fam_ids\n                      if any((m, eff, hk) in data for hk in HARNESS_ORDER)]\n            if not models:\n                continue\n            prov = \"\"\n            if fam == \"OpenAI\" and eff in (\"low\", \"medium\"):\n                prov = (' <span class=\"note\">· non-mini cells assembled from a prior '\n                        '10-trial sweep + the provider matrix (full 90 trials).</span>')\n            if fam == \"Anthropic\":\n                prov = (' <span class=\"note\">· manual “best complete run” aggregate; '\n                        'CIs recomputed as Wilson-over-90 like the other providers.</span>')\n            cap = (f'<b>{fam} · {EFFORT_WORD[fam]} = {eff}</b> · pass-rate % with '\n                   f'95% CI half-width · rows by LOC ↑ · cols small→large{prov}')\n            sec1.append(build_table(\n                columns=models,\n                col_family=MODEL_FAMILY,\n                scores_for=lambda hk, m: data.get((m, eff, hk)),\n                caption=cap,\n                vs_label=\"vs model →\",\n                col_header_fn=lambda m: MODEL_DISPLAY[m],\n            ))\n\n    # ---- Deliverable 2: transpose, one table per model ----\n    sec2 = []\n    for m in MODEL_ORDER:\n        fam = MODEL_FAMILY[m]\n        efforts = [e for e in EFFORT_ORDER[fam]\n                   if any((m, e, hk) in data for hk in HARNESS_ORDER)]\n        if not efforts:\n            continue\n        prov = \"\"\n        if fam == \"OpenAI\" and m != \"gpt-5.4-mini\":\n            prov = (' <span class=\"note\">· low/medium cells use a prior 10-trial sweep '\n                    '+ provider matrix (full 90 trials).</span>')\n        if fam == \"Anthropic\":\n            prov = (' <span class=\"note\">· manual “best complete run” aggregate.</span>')\n        erange = efforts[0] if len(efforts) == 1 else f'{efforts[0]}→{efforts[-1]}'\n        cap = (f'<b>{MODEL_DISPLAY[m]}</b> <span class=\"ci\">({fam})</span> · '\n               f'effort {erange} as columns · rows by LOC ↑{prov}')\n        sec2.append(build_table(\n            columns=efforts,\n            col_family={e: fam for e in efforts},\n            scores_for=lambda hk, e, _m=m: data.get((_m, e, hk)),\n            caption=cap,\n            vs_label=\"vs effort →\",\n            col_header_fn=lambda e: e,\n        ))\n\n    # ---- Deliverable 3: retry ablation ----\n    retry_cap = (\n        '<b>gpt-5.5 · high</b> · retry-limit ablation · pass-rate % with 95% CI '\n        'half-width · cols = max retries (r0→r3) · '\n        f'<span class=\"note\">sparkline scale {int(RETRY_LO)}–{int(RETRY_HI)}% '\n        '(tighter than the main tables) so the retry trend reads · '\n        '† mini-swe-agent r3 is the default 3-retry provider-matrix run.</span>'\n    )\n    sec3 = build_retry_table(retry, retry_cap)\n\n    # ---- Deliverable 4: TBLite, a separate benchmark, laid out on the right ----\n    sec4 = build_tblite_profile(tblite) + \"\\n\" + build_tblite_compare(tblite)\n\n    # ---- Deliverable 5: harness mechanics diff (full width, very bottom) ----\n    sec5 = build_harness_diff()\n\n    html = TEMPLATE.format(\n        scale_lo=int(SCALE_LO), scale_hi=int(SCALE_HI),\n        retry_lo=int(RETRY_LO), retry_hi=int(RETRY_HI),\n        sec1=\"\\n\".join(sec1), sec2=\"\\n\".join(sec2), sec3=sec3, sec4=sec4,\n        sec5=sec5,\n    )\n    with open(OUT, \"w\") as f:\n        f.write(html)\n    print(\"wrote\", OUT)\n\n\nTEMPLATE = \"\"\"<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n<title>TB2-core pass rates · sparkline tables</title>\n<style>\n  :root {{\n    --bg: #ffffff; --ink: #1a1a18; --tertiary: #8a8880;\n    --line: #5F5E5A; --rule: #d9d7d0; --rule-strong: #b4b1a8;\n    --hl-bg: #f3f1ea; --coral: #D85A30; --red: #c0392b;\n  }}\n  @media (prefers-color-scheme: dark) {{\n    :root {{\n      --bg: #16161a; --ink: #e9e7e0; --tertiary: #9b988e;\n      --line: #b9b6ad; --rule: #34343a; --rule-strong: #54545c;\n      --hl-bg: #232228; --coral: #ef7a4f; --red: #ff6e63;\n    }}\n  }}\n  html {{ background: var(--bg); color: var(--ink); }}\n  body {{ font-family: Georgia, 'Times New Roman', serif; max-width: 1380px;\n         margin: 40px auto; padding: 0 24px 80px; line-height: 1.5; }}\n  .page {{ display: flex; gap: 40px; align-items: flex-start; }}\n  .main {{ flex: 1 1 auto; min-width: 0; }}\n  .side {{ flex: 0 0 360px; width: 360px; }}\n  .side h2 {{ margin-top: 0; }}\n  .side figure {{ margin: 0 0 26px; }}\n  @media (max-width: 1180px) {{\n    .page {{ display: block; }}\n    .side {{ flex: none; width: auto; margin-top: 20px; }}\n  }}\n  h1 {{ font-size: 21px; margin-bottom: 2px; }}\n  h2 {{ font-size: 14px; text-transform: uppercase; letter-spacing: .09em;\n        color: var(--tertiary); font-family: system-ui, sans-serif;\n        margin: 44px 0 6px; }}\n  .sub {{ color: var(--tertiary); font-size: 13px; margin-bottom: 8px;\n          font-family: system-ui, sans-serif; }}\n  .legend {{ font-family: system-ui, sans-serif; font-size: 12px;\n             color: var(--tertiary); margin: 10px 0 26px; line-height: 1.7; }}\n  .legend b {{ color: var(--ink); }}\n  .swatch {{ display:inline-block; width:9px; height:9px; border-radius:50%;\n             vertical-align:baseline; margin: 0 1px; }}\n  figure {{ margin: 0 0 30px; }}\n  figcaption {{ font-family: system-ui, sans-serif; font-size: 12.5px;\n                color: var(--ink); margin-bottom: 7px; }}\n  figcaption .note {{ color: var(--tertiary); font-style: italic; }}\n  figcaption .ci {{ color: var(--tertiary); }}\n  table.grid {{ border-collapse: collapse; font-variant-numeric: tabular-nums;\n                font-family: system-ui, sans-serif; }}\n  .grid th, .grid td {{ border: 0.5px solid var(--rule); padding: 3px 8px;\n                        font-size: 12px; text-align: right; white-space: nowrap; }}\n  .grid thead th {{ font-weight: 600; color: var(--ink); vertical-align: bottom; }}\n  .grid th.fam {{ text-align: center; font-size: 11px; letter-spacing: .05em;\n                  text-transform: uppercase; color: var(--tertiary);\n                  border-bottom: 0.5px solid var(--rule-strong); }}\n  .grid th.corner {{ border: none; }}\n  .grid th.rowlab, .grid td.rowlab {{ text-align: left; font-weight: 400; }}\n  .grid td.rowlab {{ color: var(--ink); }}\n  .grid td.rowlab.hl {{ font-weight: 700; }}\n  .grid .ci {{ color: var(--tertiary); font-size: 10.5px; }}\n  .grid td.num {{ color: var(--ink); }}\n  .grid .miss {{ color: var(--tertiary); }}\n  .grid th.colh {{ text-align: right; }}\n  .grid th.sparkh {{ color: var(--tertiary); font-weight: 400; font-size: 11px; }}\n  .grid td.sp {{ padding: 0 4px; }}\n  .grid .famstart {{ border-left: 1.5px solid var(--rule-strong); }}\n  .grid td.sp.blank {{ border: none; }}\n  .grid tfoot td.foot {{ border-top: 0.5px solid var(--rule-strong);\n                         vertical-align: bottom; }}\n  .grid td.rowlab.foot {{ color: var(--tertiary); font-style: italic;\n                          vertical-align: bottom; }}\n  svg.spark {{ width: 88px; height: 18px; display: block; }}\n  svg.spark.tall {{ height: 90px; }}\n  .grid.cmp td.sp svg.spark {{ width: 62px; }}\n  .grid tr.minrow td.rowlab,\n  .grid tr.minrow td.num,\n  .grid tr.minrow .ci {{ color: var(--red); }}\n  p.hdiff {{ font-family: system-ui, sans-serif; font-size: 12.5px;\n             color: var(--ink); line-height: 1.6; margin: 4px 0 10px;\n             max-width: 860px; }}\n  p.hdiff code {{ font-family: ui-monospace, 'SF Mono', Menlo, monospace;\n                  font-size: 11px; }}\n  .page + h2 {{ clear: both; }}\n</style>\n</head>\n<body>\n\n<h1>TerminalBench-2 core · harness × model pass rates</h1>\n<div class=\"sub\">9-task TB2-core proxy set, 10 trials/task = 90 trials per cell.\nCrashes counted as failures (score = successes / 90).</div>\n\n<div class=\"legend\">\n  Each cell: <b>pass-rate %</b> with the 95% CI half-width in\n  <span class=\"ci\">smaller tertiary type (±pp)</span>.\n  The CI is the <b>Wilson binomial interval over the 90 trials</b> — per-trial\n  sampling noise on <i>these exact 9 tasks</i>, not the (much wider) task-level CI.\n  Main-table sparklines share one fixed score scale, <b>{scale_lo}–{scale_hi}%</b> — height\n  is comparable everywhere and a flat line means within-noise. Scores outside the band\n  rest on the floor/ceiling.\n  Row sparklines (18px) trace the row across columns;\n  the <span class=\"swatch\" style=\"background:var(--coral)\"></span> coral dot marks the\n  largest-model / highest-effort endpoint.\n  Footer sparklines (tall, 90px) trace each column down the harnesses in LOC order\n  (even-spaced, not to LOC scale); the\n  <span class=\"swatch\" style=\"background:var(--ink)\"></span> ink dot marks the\n  heaviest harness (codex-full). <b style=\"color:var(--red)\">minimal-agent</b> rows\n  are drawn in red and excluded from these vs-harness sparklines as a low outlier.\n  <b>‡</b> = msa-prompt-compressed for gpt-5.5/high is the retry-sweep r3 (default config);\n  the TB2 provider matrix did not run msa-prompt-compressed for OpenAI/DeepSeek (those cells\n  are blank). Anthropic ran both msa-fully-compressed and msa-prompt-compressed.\n  No data → “—” in the cell; the sparkline skips that point and connects straight\n  through its neighbours (the missing column is treated as absent, not zero).\n</div>\n\n<div class=\"page\">\n<div class=\"main\">\n\n<h2>I · One table per model</h2>\n<div class=\"sub\">Rows = harnesses (LOC ↑). Columns = reasoning effort (least→most).</div>\n{sec2}\n\n<h2>II · Transpose — one table per provider × effort</h2>\n<div class=\"sub\">Rows = harnesses (LOC ↑). Columns = model (small→large). Each table is a single\nprovider: OpenAI “reasoning” and DeepSeek “thinking” levels are different knobs and are\nnever placed in the same table.</div>\n{sec1}\n\n<h2>III · Retry-limit ablation</h2>\n<div class=\"sub\">gpt-5.5 · high. Rows = mini-swe-agent vs msa-prompt-compressed.\nColumns = max retries (r0→r3). Sparkline scale {retry_lo}–{retry_hi}% — local to this\nsection so the retry trend is visible; do not compare its heights to the tables above.</div>\n{sec3}\n\n</div><!-- /.main -->\n\n<aside class=\"side\">\n<h2>IV · TBLite benchmark</h2>\n<div class=\"sub\">A separate, broader benchmark — 100 tasks × 1 trial. Not the\n9-task proxy used at left.</div>\n{sec4}\n</aside>\n\n</div><!-- /.page -->\n\n<h2>V · Harness mechanics — msa-fully-compressed vs msa-prompt-compressed vs mini-swe-agent</h2>\n<div class=\"sub\">Why the rows differ in behavior, not just score. msa-prompt-compressed and\nmini-swe-agent share machinery (prompt-only diff); msa-fully-compressed is a much simpler agent.</div>\n{sec5}\n\n</body>\n</html>\n\"\"\"\n\nif __name__ == \"__main__\":\n    main()\n"}}}}