{"repo": "agentscope-ai/pawbench", "files": {"site/scripts/aggregate_results.py": {"0f794a8bb6c27aa9ee4091b2691fa30e4ed9cc8f": {"t": 1785747146.3203871, "n": 20581, "h": "4ae343513b903005", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n  format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n  ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n  ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n    python3 site/scripts/aggregate_results.py\n    python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n    \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n    \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n    # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n    # we know the official display name; users can edit this mapping.\n    \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n    \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n    if raw in MODEL_ALIAS:\n        return MODEL_ALIAS[raw]\n    head, sep, tail = raw.partition(\".\")\n    if sep and head in SAFE_VENDOR_PREFIXES and tail:\n        return tail\n    return raw\n\n\ndef canonical_harness(raw: str) -> str:\n    return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n    m = RUN_DATE_RE.search(run_name)\n    if m:\n        y, mo, d = m.groups()\n        return f\"{y}-{mo}-{d}\"\n    return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n    if not TASKS_JSON.exists():\n        print(\n            f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n            \"run build_tasks.py first.\",\n            file=sys.stderr,\n        )\n        sys.exit(2)\n    tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n    return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n    \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n    For purely-automated or purely-judge breakdowns the keys are unprefixed and\n    we leave the partition to the caller (see ``score_partition``).\n    \"\"\"\n    auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n    judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n    a = mean(auto_vals) if auto_vals else None\n    j = mean(judge_vals) if judge_vals else None\n    return a, j\n\n\ndef score_partition(\n    grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n    \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n    Tasks that don't contribute to a side return ``None`` for that side, which\n    excludes them from the corresponding column mean.\n    \"\"\"\n    if grading_type == \"hybrid\":\n        return split_breakdown(breakdown)\n    if grading_type == \"automated\":\n        return task_score, None\n    if grading_type == \"llm_judge\":\n        return None, task_score\n    # error / unknown\n    return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n    return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n    return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n    if not RESULT_ROOT.is_dir():\n        return []\n    runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n    if only:\n        wanted = set(only)\n        runs = [r for r in runs if r.name in wanted]\n    return runs\n\n\ndef _record_task(\n    *,\n    t_id: str | None,\n    score: float,\n    grading_type: str,\n    breakdown: dict[str, float],\n    status: str,\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    \"\"\"Apply a single task result to the running aggregates.\n\n    Returns ``(errored, missing_meta)`` deltas (0 or 1 each) so the caller\n    can keep cumulative counters.\n    \"\"\"\n    errored = 0\n    missing_meta = 0\n\n    if grading_type in (\"error\", \"missing\") or status in (\"error\", \"missing\"):\n        errored = 1\n        score = 0.0  # ensure error/missing tasks count as 0 toward all means\n    else:\n        a, j = score_partition(grading_type, score, breakdown)\n        if a is not None:\n            automated_scores.append(a)\n        if j is not None:\n            judge_scores.append(j)\n\n    per_task_score.append(score)\n\n    meta = task_meta.get(t_id) if t_id else None\n    if meta is None:\n        return errored, missing_meta + 1\n\n    labels = meta.get(\"labels\") or {}\n    if labels.get(\"complexity\"):\n        buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n    if labels.get(\"environment\"):\n        buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n    if labels.get(\"scenario\"):\n        sc = labels[\"scenario\"]\n        buckets[\"by_scenario\"][sc].append(score)\n        buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n    modality = labels.get(\"modality\") or {}\n    if modality.get(\"type\"):\n        buckets[\"by_modality\"][modality[\"type\"]].append(score)\n    for ch in modality.get(\"channels\") or []:\n        buckets[\"by_channel\"][ch].append(score)\n    for cap in labels.get(\"capabilities\") or []:\n        buckets[\"by_capability\"][cap].append(score)\n    if meta.get(\"source_dataset\"):\n        buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n    if meta.get(\"category\"):\n        buckets[\"by_category\"][meta[\"category\"]].append(score)\n    if meta.get(\"subcategory\"):\n        buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n    # use the *task's* original grading_type (from md), not the run-time\n    # grading_type which may be ``error``\n    orig_gt = meta.get(\"grading_type\")\n    if orig_gt:\n        buckets[\"by_grading\"][orig_gt].append(score)\n\n    return errored, missing_meta\n\n\ndef _build_task_name_index(\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n    \"\"\"Map normalised task display name → t_id for fallback lookups.\n\n    Used for the hermes single-summary format whose ``task_id`` field uses\n    internal hermes slugs that don't match our ``t_id`` system. ``task_name``\n    matches our ``name`` field exactly.\n    \"\"\"\n    index: dict[str, str] = {}\n    for t_id, meta in task_meta.items():\n        name = (meta.get(\"name\") or \"\").strip().lower()\n        if name:\n            index[name] = t_id\n    return index\n\n\ndef _build_task_slug_index(\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n    \"\"\"Map hermes ``task_id`` slugs (``core_id`` / ``task_id``) → ``t_id``.\"\"\"\n    index: dict[str, str] = {}\n    for t_id, meta in task_meta.items():\n        index[t_id.lower()] = t_id\n        for key in (\"core_id\", \"task_id\"):\n            slug = (meta.get(key) or \"\").strip().lower()\n            if slug:\n                index[slug] = t_id\n        fn = (meta.get(\"filename\") or \"\").strip()\n        if fn.endswith(\".md\"):\n            _prefix, sep, tail = fn[:-3].partition(\"_\")\n            if sep and tail:\n                index[tail.lower()] = t_id\n    return index\n\n\ndef _resolve_t_id_from_hermes_result(\n    r: dict[str, Any],\n    name_to_tid: dict[str, str],\n    slug_to_tid: dict[str, str],\n) -> str | None:\n    name_key = (r.get(\"task_name\") or \"\").strip().lower()\n    if name_key and name_key in name_to_tid:\n        return name_to_tid[name_key]\n    slug = (r.get(\"task_id\") or \"\").strip().lower()\n    if slug and slug in slug_to_tid:\n        return slug_to_tid[slug]\n    # Some hermes exports use ``T042``-style ids in ``task_id``.\n    if slug and slug.upper() in slug_to_tid:\n        return slug_to_tid[slug.upper()]\n    return None\n\n\ndef _is_pawbench_metrics(doc: dict[str, Any]) -> bool:\n    \"\"\"True when ``metrics.json`` follows the standard per-task layout.\"\"\"\n    return isinstance(doc, dict) and \"task_score\" in doc\n\n\ndef _find_hermes_summary(harness_dir: Path) -> Path | None:\n    \"\"\"Pick the hermes roll-up JSON under ``<harness>/`` (not ``workspaces/``).\"\"\"\n    best: tuple[int, Path] | None = None\n    for sf in sorted(harness_dir.glob(\"*.json\")):\n        try:\n            head = json.loads(sf.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError:\n            continue\n        if not isinstance(head, dict):\n            continue\n        results = head.get(\"results\")\n        if not isinstance(results, list) or not results:\n            continue\n        n = len(results)\n        if best is None or n > best[0]:\n            best = (n, sf)\n    return best[1] if best else None\n\n\ndef _discover_per_task_metrics(harness_dir: Path) -> list[Path]:\n    \"\"\"``T*/output/metrics.json`` files that contain pawbench ``task_score``.\"\"\"\n    out: list[Path] = []\n    for mp in sorted(harness_dir.glob(\"T*/output/metrics.json\")):\n        try:\n            doc = json.loads(mp.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError:\n            continue\n        if _is_pawbench_metrics(doc):\n            out.append(mp)\n    return out\n\n\ndef _aggregate_from_per_task_dirs(\n    metrics_files: list[Path],\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    errored = 0\n    missing_meta = 0\n    for mp in metrics_files:\n        try:\n            d = json.loads(mp.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError as exc:\n            print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n            continue\n\n        # Use the directory name as the canonical t-id source.\n        t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n        de, dm = _record_task(\n            t_id=t_id,\n            score=float(d.get(\"task_score\") or 0.0),\n            grading_type=d.get(\"grading_type\") or \"unknown\",\n            breakdown=d.get(\"breakdown\") or {},\n            status=d.get(\"status\") or \"\",\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n        errored += de\n        missing_meta += dm\n    return errored, missing_meta\n\n\ndef _aggregate_from_single_summary(\n    summary_file: Path,\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    \"\"\"Parse hermes' single-file summary format.\n\n    Schema (one JSON per ``<harness_dir>/<timestamp>.json``)::\n\n        {\n          \"summary\": {...},\n          \"results\": [\n            {\"task_id\": \"...\", \"task_name\": \"...\",\n             \"score\": float, \"max_score\": float, \"passed\": bool,\n             \"grading_type\": \"hybrid|automated|llm_judge|error\",\n             \"breakdown\": {...}, \"status\": \"success|error|timeout\",\n             \"timed_out\": bool, \"labels\": {...}},\n            ...\n          ]\n        }\n\n    The ``task_id`` field uses internal hermes slugs that don't match our\n    ``t_id`` system, so we map back via ``task_name`` against ``tasks.json``.\n    \"\"\"\n    try:\n        d = json.loads(summary_file.read_text(encoding=\"utf-8\"))\n    except json.JSONDecodeError as exc:\n        print(f\"[aggregate_results] skip {summary_file}: {exc}\", file=sys.stderr)\n        return 0, 0\n\n    results = d.get(\"results\") or []\n    if not results:\n        return 0, 0\n\n    name_to_tid = _build_task_name_index(task_meta)\n    slug_to_tid = _build_task_slug_index(task_meta)\n    errored = 0\n    missing_meta = 0\n    for r in results:\n        # ``score`` is the canonical 0-1 task score for hermes; fall back to\n        # ``task_score`` for safety in case the schema evolves.\n        score = float(r.get(\"score\") if r.get(\"score\") is not None else r.get(\"task_score\") or 0.0)\n        status = r.get(\"status\") or \"\"\n        if r.get(\"timed_out\") and status not in {\"error\", \"timeout\"}:\n            # treat timeouts as errors for accounting parity with the\n            # per-task-dir format\n            status = \"error\"\n\n        t_id = _resolve_t_id_from_hermes_result(r, name_to_tid, slug_to_tid)\n\n        de, dm = _record_task(\n            t_id=t_id,\n            score=score,\n            grading_type=r.get(\"grading_type\") or \"unknown\",\n            breakdown=r.get(\"breakdown\") or {},\n            status=status,\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n        errored += de\n        missing_meta += dm\n    return errored, missing_meta\n\n\ndef aggregate_pair(\n    run_dir: Path,\n    model_dir: Path,\n    harness_dir: Path,\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n    \"\"\"Aggregate a single ``(run, model, harness)`` triple.\n\n    Tries two on-disk layouts:\n\n    1. Per-task directories with ``T*/output/metrics.json`` (the default\n       hermes/openclaw/qwenpaw layout).\n    2. A single ``<timestamp>.json`` summary file directly under the harness\n       directory (used by some hermes runs, e.g.\n       ``result/.../openai.claude-opus-4-6/hermes/20260525_115007.json``).\n    \"\"\"\n    per_task_score: list[float] = []\n    automated_scores: list[float] = []\n    judge_scores: list[float] = []\n\n    # bucket → list of task scores (errors counted as 0)\n    buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n    metrics_files = _discover_per_task_metrics(harness_dir)\n    summary_file = _find_hermes_summary(harness_dir)\n\n    if metrics_files:\n        errored, missing_meta = _aggregate_from_per_task_dirs(\n            metrics_files,\n            task_meta,\n            per_task_score,\n            automated_scores,\n            judge_scores,\n            buckets,\n        )\n    elif summary_file is not None:\n        # Hermes roll-up export (e.g. ``openai.claude-opus-4-6/hermes/<ts>.json``)\n        # with ``results[]`` instead of per-task ``T*/output/metrics.json``.\n        errored, missing_meta = _aggregate_from_single_summary(\n            summary_file,\n            task_meta,\n            per_task_score,\n            automated_scores,\n            judge_scores,\n            buckets,\n        )\n    else:\n        return None\n\n    if not per_task_score:\n        return None\n\n    # Fill missing tasks with score=0 so the denominator is always the full\n    # task set, preventing harness crashes from inflating the mean.\n    tasks_recorded = len(per_task_score)\n    total_expected = len(task_meta)\n    missing_count = max(0, total_expected - tasks_recorded)\n\n    # For per-task dirs we can identify exactly which tasks are missing.\n    missing_tids: set[str] = set()\n    if metrics_files and missing_count > 0:\n        seen_tids = {mp.parent.parent.name.split(\"_\", 1)[0] for mp in metrics_files}\n        missing_tids = set(task_meta.keys()) - seen_tids\n    elif not metrics_files and missing_count > 0:\n        # Summary format: task_name resolution can have collisions (duplicate\n        # names across zh/en variants), so fall back to count-based fill.\n        missing_tids = set(f\"_missing_{i}\" for i in range(missing_count))\n    for t_id in missing_tids:\n        _record_task(\n            t_id=t_id,\n            score=0.0,\n            grading_type=\"missing\",\n            breakdown={},\n            status=\"missing\",\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n\n    # Compute slice means\n    by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n    return {\n        \"run\": run_dir.name,\n        \"model\": display_model(model_dir.name),\n        \"harness\": canonical_harness(harness_dir.name),\n        \"overall\": safe_mean(per_task_score),\n        \"automated\": safe_mean(automated_scores),\n        \"judge\": safe_mean(judge_scores),\n        \"tasks\": len(per_task_score),\n        \"tasks_total\": len(task_meta) or len(per_task_score),\n        \"tasks_errored\": errored,\n        \"tasks_missing\": len(missing_tids),\n        \"missing_metadata\": missing_meta,\n        \"_raw_model_dir\": model_dir.name,\n        \"_raw_harness_dir\": harness_dir.name,\n        \"updated\": run_updated(run_dir.name),\n        **by_dims,\n    }\n\n\ndef main() -> int:\n    ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n    ap.add_argument(\n        \"--run\",\n        action=\"append\",\n        default=None,\n        help=\"Restrict to specific run directory name(s); default = all under result/\",\n    )\n    ap.add_argument(\n        \"--out-dir\",\n        default=str(SUBMISSIONS_DIR),\n        help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n    )\n    args = ap.parse_args()\n\n    runs = discover_runs(args.run)\n    if not runs:\n        print(\n            f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n            file=sys.stderr,\n        )\n        return 0\n\n    task_meta = load_task_metadata()\n    out_dir = Path(args.out_dir)\n    out_dir.mkdir(parents=True, exist_ok=True)\n\n    written = 0\n    for run_dir in runs:\n        for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n            for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n                row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n                if not row:\n                    continue\n                # Use the canonical harness slug in the filename so that\n                # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n                # automatically retires the old file once the user re-runs\n                # the aggregator and removes the stale submissions.\n                fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n                out_path = out_dir / fname\n                # don't ship the internal raw-name keys into the public file\n                public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n                out_path.write_text(\n                    json.dumps(public, ensure_ascii=False, indent=2),\n                    encoding=\"utf-8\",\n                )\n                written += 1\n                print(\n                    f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s}  \"\n                    f\"overall={row['overall']:.3f}  \"\n                    f\"errored={row['tasks_errored']:>3d}  \"\n                    f\"→ {out_path.relative_to(REPO_ROOT)}\"\n                )\n\n    print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n    return 0\n\n\nif __name__ == \"__main__\":\n    raise SystemExit(main())\n"}, "5d8815ec66b03301b49accbb2d81ff20e5d52a53": {"t": 1785770709.352257, "n": 20581, "h": "4ae343513b903005", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n  format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n  ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n  ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n    python3 site/scripts/aggregate_results.py\n    python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n    \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n    \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n    # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n    # we know the official display name; users can edit this mapping.\n    \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n    \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n    if raw in MODEL_ALIAS:\n        return MODEL_ALIAS[raw]\n    head, sep, tail = raw.partition(\".\")\n    if sep and head in SAFE_VENDOR_PREFIXES and tail:\n        return tail\n    return raw\n\n\ndef canonical_harness(raw: str) -> str:\n    return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n    m = RUN_DATE_RE.search(run_name)\n    if m:\n        y, mo, d = m.groups()\n        return f\"{y}-{mo}-{d}\"\n    return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n    if not TASKS_JSON.exists():\n        print(\n            f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n            \"run build_tasks.py first.\",\n            file=sys.stderr,\n        )\n        sys.exit(2)\n    tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n    return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n    \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n    For purely-automated or purely-judge breakdowns the keys are unprefixed and\n    we leave the partition to the caller (see ``score_partition``).\n    \"\"\"\n    auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n    judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n    a = mean(auto_vals) if auto_vals else None\n    j = mean(judge_vals) if judge_vals else None\n    return a, j\n\n\ndef score_partition(\n    grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n    \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n    Tasks that don't contribute to a side return ``None`` for that side, which\n    excludes them from the corresponding column mean.\n    \"\"\"\n    if grading_type == \"hybrid\":\n        return split_breakdown(breakdown)\n    if grading_type == \"automated\":\n        return task_score, None\n    if grading_type == \"llm_judge\":\n        return None, task_score\n    # error / unknown\n    return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n    return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n    return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n    if not RESULT_ROOT.is_dir():\n        return []\n    runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n    if only:\n        wanted = set(only)\n        runs = [r for r in runs if r.name in wanted]\n    return runs\n\n\ndef _record_task(\n    *,\n    t_id: str | None,\n    score: float,\n    grading_type: str,\n    breakdown: dict[str, float],\n    status: str,\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    \"\"\"Apply a single task result to the running aggregates.\n\n    Returns ``(errored, missing_meta)`` deltas (0 or 1 each) so the caller\n    can keep cumulative counters.\n    \"\"\"\n    errored = 0\n    missing_meta = 0\n\n    if grading_type in (\"error\", \"missing\") or status in (\"error\", \"missing\"):\n        errored = 1\n        score = 0.0  # ensure error/missing tasks count as 0 toward all means\n    else:\n        a, j = score_partition(grading_type, score, breakdown)\n        if a is not None:\n            automated_scores.append(a)\n        if j is not None:\n            judge_scores.append(j)\n\n    per_task_score.append(score)\n\n    meta = task_meta.get(t_id) if t_id else None\n    if meta is None:\n        return errored, missing_meta + 1\n\n    labels = meta.get(\"labels\") or {}\n    if labels.get(\"complexity\"):\n        buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n    if labels.get(\"environment\"):\n        buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n    if labels.get(\"scenario\"):\n        sc = labels[\"scenario\"]\n        buckets[\"by_scenario\"][sc].append(score)\n        buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n    modality = labels.get(\"modality\") or {}\n    if modality.get(\"type\"):\n        buckets[\"by_modality\"][modality[\"type\"]].append(score)\n    for ch in modality.get(\"channels\") or []:\n        buckets[\"by_channel\"][ch].append(score)\n    for cap in labels.get(\"capabilities\") or []:\n        buckets[\"by_capability\"][cap].append(score)\n    if meta.get(\"source_dataset\"):\n        buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n    if meta.get(\"category\"):\n        buckets[\"by_category\"][meta[\"category\"]].append(score)\n    if meta.get(\"subcategory\"):\n        buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n    # use the *task's* original grading_type (from md), not the run-time\n    # grading_type which may be ``error``\n    orig_gt = meta.get(\"grading_type\")\n    if orig_gt:\n        buckets[\"by_grading\"][orig_gt].append(score)\n\n    return errored, missing_meta\n\n\ndef _build_task_name_index(\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n    \"\"\"Map normalised task display name → t_id for fallback lookups.\n\n    Used for the hermes single-summary format whose ``task_id`` field uses\n    internal hermes slugs that don't match our ``t_id`` system. ``task_name``\n    matches our ``name`` field exactly.\n    \"\"\"\n    index: dict[str, str] = {}\n    for t_id, meta in task_meta.items():\n        name = (meta.get(\"name\") or \"\").strip().lower()\n        if name:\n            index[name] = t_id\n    return index\n\n\ndef _build_task_slug_index(\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n    \"\"\"Map hermes ``task_id`` slugs (``core_id`` / ``task_id``) → ``t_id``.\"\"\"\n    index: dict[str, str] = {}\n    for t_id, meta in task_meta.items():\n        index[t_id.lower()] = t_id\n        for key in (\"core_id\", \"task_id\"):\n            slug = (meta.get(key) or \"\").strip().lower()\n            if slug:\n                index[slug] = t_id\n        fn = (meta.get(\"filename\") or \"\").strip()\n        if fn.endswith(\".md\"):\n            _prefix, sep, tail = fn[:-3].partition(\"_\")\n            if sep and tail:\n                index[tail.lower()] = t_id\n    return index\n\n\ndef _resolve_t_id_from_hermes_result(\n    r: dict[str, Any],\n    name_to_tid: dict[str, str],\n    slug_to_tid: dict[str, str],\n) -> str | None:\n    name_key = (r.get(\"task_name\") or \"\").strip().lower()\n    if name_key and name_key in name_to_tid:\n        return name_to_tid[name_key]\n    slug = (r.get(\"task_id\") or \"\").strip().lower()\n    if slug and slug in slug_to_tid:\n        return slug_to_tid[slug]\n    # Some hermes exports use ``T042``-style ids in ``task_id``.\n    if slug and slug.upper() in slug_to_tid:\n        return slug_to_tid[slug.upper()]\n    return None\n\n\ndef _is_pawbench_metrics(doc: dict[str, Any]) -> bool:\n    \"\"\"True when ``metrics.json`` follows the standard per-task layout.\"\"\"\n    return isinstance(doc, dict) and \"task_score\" in doc\n\n\ndef _find_hermes_summary(harness_dir: Path) -> Path | None:\n    \"\"\"Pick the hermes roll-up JSON under ``<harness>/`` (not ``workspaces/``).\"\"\"\n    best: tuple[int, Path] | None = None\n    for sf in sorted(harness_dir.glob(\"*.json\")):\n        try:\n            head = json.loads(sf.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError:\n            continue\n        if not isinstance(head, dict):\n            continue\n        results = head.get(\"results\")\n        if not isinstance(results, list) or not results:\n            continue\n        n = len(results)\n        if best is None or n > best[0]:\n            best = (n, sf)\n    return best[1] if best else None\n\n\ndef _discover_per_task_metrics(harness_dir: Path) -> list[Path]:\n    \"\"\"``T*/output/metrics.json`` files that contain pawbench ``task_score``.\"\"\"\n    out: list[Path] = []\n    for mp in sorted(harness_dir.glob(\"T*/output/metrics.json\")):\n        try:\n            doc = json.loads(mp.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError:\n            continue\n        if _is_pawbench_metrics(doc):\n            out.append(mp)\n    return out\n\n\ndef _aggregate_from_per_task_dirs(\n    metrics_files: list[Path],\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    errored = 0\n    missing_meta = 0\n    for mp in metrics_files:\n        try:\n            d = json.loads(mp.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError as exc:\n            print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n            continue\n\n        # Use the directory name as the canonical t-id source.\n        t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n        de, dm = _record_task(\n            t_id=t_id,\n            score=float(d.get(\"task_score\") or 0.0),\n            grading_type=d.get(\"grading_type\") or \"unknown\",\n            breakdown=d.get(\"breakdown\") or {},\n            status=d.get(\"status\") or \"\",\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n        errored += de\n        missing_meta += dm\n    return errored, missing_meta\n\n\ndef _aggregate_from_single_summary(\n    summary_file: Path,\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    \"\"\"Parse hermes' single-file summary format.\n\n    Schema (one JSON per ``<harness_dir>/<timestamp>.json``)::\n\n        {\n          \"summary\": {...},\n          \"results\": [\n            {\"task_id\": \"...\", \"task_name\": \"...\",\n             \"score\": float, \"max_score\": float, \"passed\": bool,\n             \"grading_type\": \"hybrid|automated|llm_judge|error\",\n             \"breakdown\": {...}, \"status\": \"success|error|timeout\",\n             \"timed_out\": bool, \"labels\": {...}},\n            ...\n          ]\n        }\n\n    The ``task_id`` field uses internal hermes slugs that don't match our\n    ``t_id`` system, so we map back via ``task_name`` against ``tasks.json``.\n    \"\"\"\n    try:\n        d = json.loads(summary_file.read_text(encoding=\"utf-8\"))\n    except json.JSONDecodeError as exc:\n        print(f\"[aggregate_results] skip {summary_file}: {exc}\", file=sys.stderr)\n        return 0, 0\n\n    results = d.get(\"results\") or []\n    if not results:\n        return 0, 0\n\n    name_to_tid = _build_task_name_index(task_meta)\n    slug_to_tid = _build_task_slug_index(task_meta)\n    errored = 0\n    missing_meta = 0\n    for r in results:\n        # ``score`` is the canonical 0-1 task score for hermes; fall back to\n        # ``task_score`` for safety in case the schema evolves.\n        score = float(r.get(\"score\") if r.get(\"score\") is not None else r.get(\"task_score\") or 0.0)\n        status = r.get(\"status\") or \"\"\n        if r.get(\"timed_out\") and status not in {\"error\", \"timeout\"}:\n            # treat timeouts as errors for accounting parity with the\n            # per-task-dir format\n            status = \"error\"\n\n        t_id = _resolve_t_id_from_hermes_result(r, name_to_tid, slug_to_tid)\n\n        de, dm = _record_task(\n            t_id=t_id,\n            score=score,\n            grading_type=r.get(\"grading_type\") or \"unknown\",\n            breakdown=r.get(\"breakdown\") or {},\n            status=status,\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n        errored += de\n        missing_meta += dm\n    return errored, missing_meta\n\n\ndef aggregate_pair(\n    run_dir: Path,\n    model_dir: Path,\n    harness_dir: Path,\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n    \"\"\"Aggregate a single ``(run, model, harness)`` triple.\n\n    Tries two on-disk layouts:\n\n    1. Per-task directories with ``T*/output/metrics.json`` (the default\n       hermes/openclaw/qwenpaw layout).\n    2. A single ``<timestamp>.json`` summary file directly under the harness\n       directory (used by some hermes runs, e.g.\n       ``result/.../openai.claude-opus-4-6/hermes/20260525_115007.json``).\n    \"\"\"\n    per_task_score: list[float] = []\n    automated_scores: list[float] = []\n    judge_scores: list[float] = []\n\n    # bucket → list of task scores (errors counted as 0)\n    buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n    metrics_files = _discover_per_task_metrics(harness_dir)\n    summary_file = _find_hermes_summary(harness_dir)\n\n    if metrics_files:\n        errored, missing_meta = _aggregate_from_per_task_dirs(\n            metrics_files,\n            task_meta,\n            per_task_score,\n            automated_scores,\n            judge_scores,\n            buckets,\n        )\n    elif summary_file is not None:\n        # Hermes roll-up export (e.g. ``openai.claude-opus-4-6/hermes/<ts>.json``)\n        # with ``results[]`` instead of per-task ``T*/output/metrics.json``.\n        errored, missing_meta = _aggregate_from_single_summary(\n            summary_file,\n            task_meta,\n            per_task_score,\n            automated_scores,\n            judge_scores,\n            buckets,\n        )\n    else:\n        return None\n\n    if not per_task_score:\n        return None\n\n    # Fill missing tasks with score=0 so the denominator is always the full\n    # task set, preventing harness crashes from inflating the mean.\n    tasks_recorded = len(per_task_score)\n    total_expected = len(task_meta)\n    missing_count = max(0, total_expected - tasks_recorded)\n\n    # For per-task dirs we can identify exactly which tasks are missing.\n    missing_tids: set[str] = set()\n    if metrics_files and missing_count > 0:\n        seen_tids = {mp.parent.parent.name.split(\"_\", 1)[0] for mp in metrics_files}\n        missing_tids = set(task_meta.keys()) - seen_tids\n    elif not metrics_files and missing_count > 0:\n        # Summary format: task_name resolution can have collisions (duplicate\n        # names across zh/en variants), so fall back to count-based fill.\n        missing_tids = set(f\"_missing_{i}\" for i in range(missing_count))\n    for t_id in missing_tids:\n        _record_task(\n            t_id=t_id,\n            score=0.0,\n            grading_type=\"missing\",\n            breakdown={},\n            status=\"missing\",\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n\n    # Compute slice means\n    by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n    return {\n        \"run\": run_dir.name,\n        \"model\": display_model(model_dir.name),\n        \"harness\": canonical_harness(harness_dir.name),\n        \"overall\": safe_mean(per_task_score),\n        \"automated\": safe_mean(automated_scores),\n        \"judge\": safe_mean(judge_scores),\n        \"tasks\": len(per_task_score),\n        \"tasks_total\": len(task_meta) or len(per_task_score),\n        \"tasks_errored\": errored,\n        \"tasks_missing\": len(missing_tids),\n        \"missing_metadata\": missing_meta,\n        \"_raw_model_dir\": model_dir.name,\n        \"_raw_harness_dir\": harness_dir.name,\n        \"updated\": run_updated(run_dir.name),\n        **by_dims,\n    }\n\n\ndef main() -> int:\n    ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n    ap.add_argument(\n        \"--run\",\n        action=\"append\",\n        default=None,\n        help=\"Restrict to specific run directory name(s); default = all under result/\",\n    )\n    ap.add_argument(\n        \"--out-dir\",\n        default=str(SUBMISSIONS_DIR),\n        help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n    )\n    args = ap.parse_args()\n\n    runs = discover_runs(args.run)\n    if not runs:\n        print(\n            f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n            file=sys.stderr,\n        )\n        return 0\n\n    task_meta = load_task_metadata()\n    out_dir = Path(args.out_dir)\n    out_dir.mkdir(parents=True, exist_ok=True)\n\n    written = 0\n    for run_dir in runs:\n        for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n            for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n                row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n                if not row:\n                    continue\n                # Use the canonical harness slug in the filename so that\n                # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n                # automatically retires the old file once the user re-runs\n                # the aggregator and removes the stale submissions.\n                fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n                out_path = out_dir / fname\n                # don't ship the internal raw-name keys into the public file\n                public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n                out_path.write_text(\n                    json.dumps(public, ensure_ascii=False, indent=2),\n                    encoding=\"utf-8\",\n                )\n                written += 1\n                print(\n                    f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s}  \"\n                    f\"overall={row['overall']:.3f}  \"\n                    f\"errored={row['tasks_errored']:>3d}  \"\n                    f\"→ {out_path.relative_to(REPO_ROOT)}\"\n                )\n\n    print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n    return 0\n\n\nif __name__ == \"__main__\":\n    raise SystemExit(main())\n"}, "c9dc00a821f4c89b3cadc6ec436fc81e5a3d1f2e": {"t": 1785770710.666815, "n": 19259, "h": "c96cefdd0a9e3666", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n  format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n  ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n  ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n    python3 site/scripts/aggregate_results.py\n    python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n    \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n    \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n    # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n    # we know the official display name; users can edit this mapping.\n    \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n    \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n    if raw in MODEL_ALIAS:\n        return MODEL_ALIAS[raw]\n    head, sep, tail = raw.partition(\".\")\n    if sep and head in SAFE_VENDOR_PREFIXES and tail:\n        return tail\n    return raw\n\n\ndef canonical_harness(raw: str) -> str:\n    return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n    m = RUN_DATE_RE.search(run_name)\n    if m:\n        y, mo, d = m.groups()\n        return f\"{y}-{mo}-{d}\"\n    return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n    if not TASKS_JSON.exists():\n        print(\n            f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n            \"run build_tasks.py first.\",\n            file=sys.stderr,\n        )\n        sys.exit(2)\n    tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n    return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n    \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n    For purely-automated or purely-judge breakdowns the keys are unprefixed and\n    we leave the partition to the caller (see ``score_partition``).\n    \"\"\"\n    auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n    judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n    a = mean(auto_vals) if auto_vals else None\n    j = mean(judge_vals) if judge_vals else None\n    return a, j\n\n\ndef score_partition(\n    grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n    \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n    Tasks that don't contribute to a side return ``None`` for that side, which\n    excludes them from the corresponding column mean.\n    \"\"\"\n    if grading_type == \"hybrid\":\n        return split_breakdown(breakdown)\n    if grading_type == \"automated\":\n        return task_score, None\n    if grading_type == \"llm_judge\":\n        return None, task_score\n    # error / unknown\n    return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n    return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n    return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n    if not RESULT_ROOT.is_dir():\n        return []\n    runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n    if only:\n        wanted = set(only)\n        runs = [r for r in runs if r.name in wanted]\n    return runs\n\n\ndef _record_task(\n    *,\n    t_id: str | None,\n    score: float,\n    grading_type: str,\n    breakdown: dict[str, float],\n    status: str,\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    \"\"\"Apply a single task result to the running aggregates.\n\n    Returns ``(errored, missing_meta)`` deltas (0 or 1 each) so the caller\n    can keep cumulative counters.\n    \"\"\"\n    errored = 0\n    missing_meta = 0\n\n    if grading_type == \"error\" or status == \"error\":\n        errored = 1\n        score = 0.0  # ensure error tasks count as 0 toward all means\n    else:\n        a, j = score_partition(grading_type, score, breakdown)\n        if a is not None:\n            automated_scores.append(a)\n        if j is not None:\n            judge_scores.append(j)\n\n    per_task_score.append(score)\n\n    meta = task_meta.get(t_id) if t_id else None\n    if meta is None:\n        return errored, missing_meta + 1\n\n    labels = meta.get(\"labels\") or {}\n    if labels.get(\"complexity\"):\n        buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n    if labels.get(\"environment\"):\n        buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n    if labels.get(\"scenario\"):\n        sc = labels[\"scenario\"]\n        buckets[\"by_scenario\"][sc].append(score)\n        buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n    modality = labels.get(\"modality\") or {}\n    if modality.get(\"type\"):\n        buckets[\"by_modality\"][modality[\"type\"]].append(score)\n    for ch in modality.get(\"channels\") or []:\n        buckets[\"by_channel\"][ch].append(score)\n    for cap in labels.get(\"capabilities\") or []:\n        buckets[\"by_capability\"][cap].append(score)\n    if meta.get(\"source_dataset\"):\n        buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n    if meta.get(\"category\"):\n        buckets[\"by_category\"][meta[\"category\"]].append(score)\n    if meta.get(\"subcategory\"):\n        buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n    # use the *task's* original grading_type (from md), not the run-time\n    # grading_type which may be ``error``\n    orig_gt = meta.get(\"grading_type\")\n    if orig_gt:\n        buckets[\"by_grading\"][orig_gt].append(score)\n\n    return errored, missing_meta\n\n\ndef _build_task_name_index(\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n    \"\"\"Map normalised task display name → t_id for fallback lookups.\n\n    Used for the hermes single-summary format whose ``task_id`` field uses\n    internal hermes slugs that don't match our ``t_id`` system. ``task_name``\n    matches our ``name`` field exactly.\n    \"\"\"\n    index: dict[str, str] = {}\n    for t_id, meta in task_meta.items():\n        name = (meta.get(\"name\") or \"\").strip().lower()\n        if name:\n            index[name] = t_id\n    return index\n\n\ndef _build_task_slug_index(\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n    \"\"\"Map hermes ``task_id`` slugs (``core_id`` / ``task_id``) → ``t_id``.\"\"\"\n    index: dict[str, str] = {}\n    for t_id, meta in task_meta.items():\n        index[t_id.lower()] = t_id\n        for key in (\"core_id\", \"task_id\"):\n            slug = (meta.get(key) or \"\").strip().lower()\n            if slug:\n                index[slug] = t_id\n        fn = (meta.get(\"filename\") or \"\").strip()\n        if fn.endswith(\".md\"):\n            _prefix, sep, tail = fn[:-3].partition(\"_\")\n            if sep and tail:\n                index[tail.lower()] = t_id\n    return index\n\n\ndef _resolve_t_id_from_hermes_result(\n    r: dict[str, Any],\n    name_to_tid: dict[str, str],\n    slug_to_tid: dict[str, str],\n) -> str | None:\n    name_key = (r.get(\"task_name\") or \"\").strip().lower()\n    if name_key and name_key in name_to_tid:\n        return name_to_tid[name_key]\n    slug = (r.get(\"task_id\") or \"\").strip().lower()\n    if slug and slug in slug_to_tid:\n        return slug_to_tid[slug]\n    # Some hermes exports use ``T042``-style ids in ``task_id``.\n    if slug and slug.upper() in slug_to_tid:\n        return slug_to_tid[slug.upper()]\n    return None\n\n\ndef _is_pawbench_metrics(doc: dict[str, Any]) -> bool:\n    \"\"\"True when ``metrics.json`` follows the standard per-task layout.\"\"\"\n    return isinstance(doc, dict) and \"task_score\" in doc\n\n\ndef _find_hermes_summary(harness_dir: Path) -> Path | None:\n    \"\"\"Pick the hermes roll-up JSON under ``<harness>/`` (not ``workspaces/``).\"\"\"\n    best: tuple[int, Path] | None = None\n    for sf in sorted(harness_dir.glob(\"*.json\")):\n        try:\n            head = json.loads(sf.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError:\n            continue\n        if not isinstance(head, dict):\n            continue\n        results = head.get(\"results\")\n        if not isinstance(results, list) or not results:\n            continue\n        n = len(results)\n        if best is None or n > best[0]:\n            best = (n, sf)\n    return best[1] if best else None\n\n\ndef _discover_per_task_metrics(harness_dir: Path) -> list[Path]:\n    \"\"\"``T*/output/metrics.json`` files that contain pawbench ``task_score``.\"\"\"\n    out: list[Path] = []\n    for mp in sorted(harness_dir.glob(\"T*/output/metrics.json\")):\n        try:\n            doc = json.loads(mp.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError:\n            continue\n        if _is_pawbench_metrics(doc):\n            out.append(mp)\n    return out\n\n\ndef _aggregate_from_per_task_dirs(\n    metrics_files: list[Path],\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    errored = 0\n    missing_meta = 0\n    for mp in metrics_files:\n        try:\n            d = json.loads(mp.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError as exc:\n            print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n            continue\n\n        # Use the directory name as the canonical t-id source.\n        t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n        de, dm = _record_task(\n            t_id=t_id,\n            score=float(d.get(\"task_score\") or 0.0),\n            grading_type=d.get(\"grading_type\") or \"unknown\",\n            breakdown=d.get(\"breakdown\") or {},\n            status=d.get(\"status\") or \"\",\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n        errored += de\n        missing_meta += dm\n    return errored, missing_meta\n\n\ndef _aggregate_from_single_summary(\n    summary_file: Path,\n    task_meta: dict[str, dict[str, Any]],\n    per_task_score: list[float],\n    automated_scores: list[float],\n    judge_scores: list[float],\n    buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n    \"\"\"Parse hermes' single-file summary format.\n\n    Schema (one JSON per ``<harness_dir>/<timestamp>.json``)::\n\n        {\n          \"summary\": {...},\n          \"results\": [\n            {\"task_id\": \"...\", \"task_name\": \"...\",\n             \"score\": float, \"max_score\": float, \"passed\": bool,\n             \"grading_type\": \"hybrid|automated|llm_judge|error\",\n             \"breakdown\": {...}, \"status\": \"success|error|timeout\",\n             \"timed_out\": bool, \"labels\": {...}},\n            ...\n          ]\n        }\n\n    The ``task_id`` field uses internal hermes slugs that don't match our\n    ``t_id`` system, so we map back via ``task_name`` against ``tasks.json``.\n    \"\"\"\n    try:\n        d = json.loads(summary_file.read_text(encoding=\"utf-8\"))\n    except json.JSONDecodeError as exc:\n        print(f\"[aggregate_results] skip {summary_file}: {exc}\", file=sys.stderr)\n        return 0, 0\n\n    results = d.get(\"results\") or []\n    if not results:\n        return 0, 0\n\n    name_to_tid = _build_task_name_index(task_meta)\n    slug_to_tid = _build_task_slug_index(task_meta)\n    errored = 0\n    missing_meta = 0\n    for r in results:\n        # ``score`` is the canonical 0-1 task score for hermes; fall back to\n        # ``task_score`` for safety in case the schema evolves.\n        score = float(r.get(\"score\") if r.get(\"score\") is not None else r.get(\"task_score\") or 0.0)\n        status = r.get(\"status\") or \"\"\n        if r.get(\"timed_out\") and status not in {\"error\", \"timeout\"}:\n            # treat timeouts as errors for accounting parity with the\n            # per-task-dir format\n            status = \"error\"\n\n        t_id = _resolve_t_id_from_hermes_result(r, name_to_tid, slug_to_tid)\n\n        de, dm = _record_task(\n            t_id=t_id,\n            score=score,\n            grading_type=r.get(\"grading_type\") or \"unknown\",\n            breakdown=r.get(\"breakdown\") or {},\n            status=status,\n            task_meta=task_meta,\n            per_task_score=per_task_score,\n            automated_scores=automated_scores,\n            judge_scores=judge_scores,\n            buckets=buckets,\n        )\n        errored += de\n        missing_meta += dm\n    return errored, missing_meta\n\n\ndef aggregate_pair(\n    run_dir: Path,\n    model_dir: Path,\n    harness_dir: Path,\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n    \"\"\"Aggregate a single ``(run, model, harness)`` triple.\n\n    Tries two on-disk layouts:\n\n    1. Per-task directories with ``T*/output/metrics.json`` (the default\n       hermes/openclaw/qwenpaw layout).\n    2. A single ``<timestamp>.json`` summary file directly under the harness\n       directory (used by some hermes runs, e.g.\n       ``result/.../openai.claude-opus-4-6/hermes/20260525_115007.json``).\n    \"\"\"\n    per_task_score: list[float] = []\n    automated_scores: list[float] = []\n    judge_scores: list[float] = []\n\n    # bucket → list of task scores (errors counted as 0)\n    buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n    metrics_files = _discover_per_task_metrics(harness_dir)\n    summary_file = _find_hermes_summary(harness_dir)\n\n    if metrics_files:\n        errored, missing_meta = _aggregate_from_per_task_dirs(\n            metrics_files,\n            task_meta,\n            per_task_score,\n            automated_scores,\n            judge_scores,\n            buckets,\n        )\n    elif summary_file is not None:\n        # Hermes roll-up export (e.g. ``openai.claude-opus-4-6/hermes/<ts>.json``)\n        # with ``results[]`` instead of per-task ``T*/output/metrics.json``.\n        errored, missing_meta = _aggregate_from_single_summary(\n            summary_file,\n            task_meta,\n            per_task_score,\n            automated_scores,\n            judge_scores,\n            buckets,\n        )\n    else:\n        return None\n\n    if not per_task_score:\n        return None\n\n    # Compute slice means\n    by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n    return {\n        \"run\": run_dir.name,\n        \"model\": display_model(model_dir.name),\n        \"harness\": canonical_harness(harness_dir.name),\n        \"overall\": safe_mean(per_task_score),\n        \"automated\": safe_mean(automated_scores),\n        \"judge\": safe_mean(judge_scores),\n        \"tasks\": len(per_task_score),\n        \"tasks_total\": len(task_meta) or len(per_task_score),\n        \"tasks_errored\": errored,\n        \"missing_metadata\": missing_meta,\n        \"_raw_model_dir\": model_dir.name,\n        \"_raw_harness_dir\": harness_dir.name,\n        \"updated\": run_updated(run_dir.name),\n        **by_dims,\n    }\n\n\ndef main() -> int:\n    ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n    ap.add_argument(\n        \"--run\",\n        action=\"append\",\n        default=None,\n        help=\"Restrict to specific run directory name(s); default = all under result/\",\n    )\n    ap.add_argument(\n        \"--out-dir\",\n        default=str(SUBMISSIONS_DIR),\n        help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n    )\n    args = ap.parse_args()\n\n    runs = discover_runs(args.run)\n    if not runs:\n        print(\n            f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n            file=sys.stderr,\n        )\n        return 0\n\n    task_meta = load_task_metadata()\n    out_dir = Path(args.out_dir)\n    out_dir.mkdir(parents=True, exist_ok=True)\n\n    written = 0\n    for run_dir in runs:\n        for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n            for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n                row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n                if not row:\n                    continue\n                # Use the canonical harness slug in the filename so that\n                # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n                # automatically retires the old file once the user re-runs\n                # the aggregator and removes the stale submissions.\n                fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n                out_path = out_dir / fname\n                # don't ship the internal raw-name keys into the public file\n                public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n                out_path.write_text(\n                    json.dumps(public, ensure_ascii=False, indent=2),\n                    encoding=\"utf-8\",\n                )\n                written += 1\n                print(\n                    f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s}  \"\n                    f\"overall={row['overall']:.3f}  \"\n                    f\"errored={row['tasks_errored']:>3d}  \"\n                    f\"→ {out_path.relative_to(REPO_ROOT)}\"\n                )\n\n    print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n    return 0\n\n\nif __name__ == \"__main__\":\n    raise SystemExit(main())\n"}, "d40f2e9686710323df316efcd2fa179dd751be60": {"t": 1785770712.0188928, "n": 11663, "h": "cb87a7c3736d63ba", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n  format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n  ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n  ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n    python3 site/scripts/aggregate_results.py\n    python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n    \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n    \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n    # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n    # we know the official display name; users can edit this mapping.\n    \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n    \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n    if raw in MODEL_ALIAS:\n        return MODEL_ALIAS[raw]\n    head, sep, tail = raw.partition(\".\")\n    if sep and head in SAFE_VENDOR_PREFIXES and tail:\n        return tail\n    return raw\n\n\ndef canonical_harness(raw: str) -> str:\n    return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n    m = RUN_DATE_RE.search(run_name)\n    if m:\n        y, mo, d = m.groups()\n        return f\"{y}-{mo}-{d}\"\n    return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n    if not TASKS_JSON.exists():\n        print(\n            f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n            \"run build_tasks.py first.\",\n            file=sys.stderr,\n        )\n        sys.exit(2)\n    tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n    return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n    \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n    For purely-automated or purely-judge breakdowns the keys are unprefixed and\n    we leave the partition to the caller (see ``score_partition``).\n    \"\"\"\n    auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n    judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n    a = mean(auto_vals) if auto_vals else None\n    j = mean(judge_vals) if judge_vals else None\n    return a, j\n\n\ndef score_partition(\n    grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n    \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n    Tasks that don't contribute to a side return ``None`` for that side, which\n    excludes them from the corresponding column mean.\n    \"\"\"\n    if grading_type == \"hybrid\":\n        return split_breakdown(breakdown)\n    if grading_type == \"automated\":\n        return task_score, None\n    if grading_type == \"llm_judge\":\n        return None, task_score\n    # error / unknown\n    return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n    return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n    return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n    if not RESULT_ROOT.is_dir():\n        return []\n    runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n    if only:\n        wanted = set(only)\n        runs = [r for r in runs if r.name in wanted]\n    return runs\n\n\ndef aggregate_pair(\n    run_dir: Path,\n    model_dir: Path,\n    harness_dir: Path,\n    task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n    \"\"\"Aggregate a single ``(run, model, harness)`` triple.\"\"\"\n    metrics_files = sorted(harness_dir.glob(\"T*/output/metrics.json\"))\n    if not metrics_files:\n        return None\n\n    per_task_score: list[float] = []\n    automated_scores: list[float] = []\n    judge_scores: list[float] = []\n    errored = 0\n    missing_meta = 0\n\n    # bucket → list of task scores (errors counted as 0)\n    buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n    seen_ids: set[str] = set()\n    for mp in metrics_files:\n        try:\n            d = json.loads(mp.read_text(encoding=\"utf-8\"))\n        except json.JSONDecodeError as exc:\n            print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n            continue\n\n        # Use the directory name as the canonical t-id source; fall back to the\n        # ``instance_id`` or the dir prefix when missing.\n        t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n        seen_ids.add(t_id)\n\n        score = float(d.get(\"task_score\") or 0.0)\n        gt = d.get(\"grading_type\") or \"unknown\"\n        breakdown = d.get(\"breakdown\") or {}\n\n        if gt == \"error\" or d.get(\"status\") == \"error\":\n            errored += 1\n            score = 0.0  # ensure error tasks count as 0 toward all means\n        else:\n            a, j = score_partition(gt, score, breakdown)\n            if a is not None:\n                automated_scores.append(a)\n            if j is not None:\n                judge_scores.append(j)\n\n        per_task_score.append(score)\n\n        meta = task_meta.get(t_id)\n        if meta is None:\n            missing_meta += 1\n            continue\n\n        labels = meta.get(\"labels\") or {}\n        # single-valued buckets\n        if labels.get(\"complexity\"):\n            buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n        if labels.get(\"environment\"):\n            buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n        if labels.get(\"scenario\"):\n            sc = labels[\"scenario\"]\n            buckets[\"by_scenario\"][sc].append(score)\n            buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n        modality = labels.get(\"modality\") or {}\n        if modality.get(\"type\"):\n            buckets[\"by_modality\"][modality[\"type\"]].append(score)\n        for ch in modality.get(\"channels\") or []:\n            buckets[\"by_channel\"][ch].append(score)\n        for cap in labels.get(\"capabilities\") or []:\n            buckets[\"by_capability\"][cap].append(score)\n        if meta.get(\"source_dataset\"):\n            buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n        if meta.get(\"category\"):\n            buckets[\"by_category\"][meta[\"category\"]].append(score)\n        if meta.get(\"subcategory\"):\n            buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n        # use the *task's* original grading_type (from md), not the run-time\n        # grading_type which may be ``error``\n        orig_gt = meta.get(\"grading_type\")\n        if orig_gt:\n            buckets[\"by_grading\"][orig_gt].append(score)\n\n    if not per_task_score:\n        return None\n\n    # Compute slice means\n    by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n    return {\n        \"run\": run_dir.name,\n        \"model\": display_model(model_dir.name),\n        \"harness\": canonical_harness(harness_dir.name),\n        \"overall\": safe_mean(per_task_score),\n        \"automated\": safe_mean(automated_scores),\n        \"judge\": safe_mean(judge_scores),\n        \"tasks\": len(per_task_score),\n        \"tasks_total\": len(task_meta) or len(per_task_score),\n        \"tasks_errored\": errored,\n        \"missing_metadata\": missing_meta,\n        \"_raw_model_dir\": model_dir.name,\n        \"_raw_harness_dir\": harness_dir.name,\n        \"updated\": run_updated(run_dir.name),\n        **by_dims,\n    }\n\n\ndef main() -> int:\n    ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n    ap.add_argument(\n        \"--run\",\n        action=\"append\",\n        default=None,\n        help=\"Restrict to specific run directory name(s); default = all under result/\",\n    )\n    ap.add_argument(\n        \"--out-dir\",\n        default=str(SUBMISSIONS_DIR),\n        help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n    )\n    args = ap.parse_args()\n\n    runs = discover_runs(args.run)\n    if not runs:\n        print(\n            f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n            file=sys.stderr,\n        )\n        return 0\n\n    task_meta = load_task_metadata()\n    out_dir = Path(args.out_dir)\n    out_dir.mkdir(parents=True, exist_ok=True)\n\n    written = 0\n    for run_dir in runs:\n        for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n            for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n                row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n                if not row:\n                    continue\n                # Use the canonical harness slug in the filename so that\n                # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n                # automatically retires the old file once the user re-runs\n                # the aggregator and removes the stale submissions.\n                fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n                out_path = out_dir / fname\n                # don't ship the internal raw-name keys into the public file\n                public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n                out_path.write_text(\n                    json.dumps(public, ensure_ascii=False, indent=2),\n                    encoding=\"utf-8\",\n                )\n                written += 1\n                print(\n                    f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s}  \"\n                    f\"overall={row['overall']:.3f}  \"\n                    f\"errored={row['tasks_errored']:>3d}  \"\n                    f\"→ {out_path.relative_to(REPO_ROOT)}\"\n                )\n\n    print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n    return 0\n\n\nif __name__ == \"__main__\":\n    raise SystemExit(main())\n"}}}}