1 line
74 KiB
JSON
1 line
74 KiB
JSON
{"repo": "agentscope-ai/pawbench", "files": {"site/scripts/aggregate_results.py": {"0f794a8bb6c27aa9ee4091b2691fa30e4ed9cc8f": {"t": 1785747146.3203871, "n": 20581, "h": "4ae343513b903005", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n python3 site/scripts/aggregate_results.py\n python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n # we know the official display name; users can edit this mapping.\n \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n if raw in MODEL_ALIAS:\n return MODEL_ALIAS[raw]\n head, sep, tail = raw.partition(\".\")\n if sep and head in SAFE_VENDOR_PREFIXES and tail:\n return tail\n return raw\n\n\ndef canonical_harness(raw: str) -> str:\n return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n m = RUN_DATE_RE.search(run_name)\n if m:\n y, mo, d = m.groups()\n return f\"{y}-{mo}-{d}\"\n return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n if not TASKS_JSON.exists():\n print(\n f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n \"run build_tasks.py first.\",\n file=sys.stderr,\n )\n sys.exit(2)\n tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n For purely-automated or purely-judge breakdowns the keys are unprefixed and\n we leave the partition to the caller (see ``score_partition``).\n \"\"\"\n auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n a = mean(auto_vals) if auto_vals else None\n j = mean(judge_vals) if judge_vals else None\n return a, j\n\n\ndef score_partition(\n grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n Tasks that don't contribute to a side return ``None`` for that side, which\n excludes them from the corresponding column mean.\n \"\"\"\n if grading_type == \"hybrid\":\n return split_breakdown(breakdown)\n if grading_type == \"automated\":\n return task_score, None\n if grading_type == \"llm_judge\":\n return None, task_score\n # error / unknown\n return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n if not RESULT_ROOT.is_dir():\n return []\n runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n if only:\n wanted = set(only)\n runs = [r for r in runs if r.name in wanted]\n return runs\n\n\ndef _record_task(\n *,\n t_id: str | None,\n score: float,\n grading_type: str,\n breakdown: dict[str, float],\n status: str,\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n \"\"\"Apply a single task result to the running aggregates.\n\n Returns ``(errored, missing_meta)`` deltas (0 or 1 each) so the caller\n can keep cumulative counters.\n \"\"\"\n errored = 0\n missing_meta = 0\n\n if grading_type in (\"error\", \"missing\") or status in (\"error\", \"missing\"):\n errored = 1\n score = 0.0 # ensure error/missing tasks count as 0 toward all means\n else:\n a, j = score_partition(grading_type, score, breakdown)\n if a is not None:\n automated_scores.append(a)\n if j is not None:\n judge_scores.append(j)\n\n per_task_score.append(score)\n\n meta = task_meta.get(t_id) if t_id else None\n if meta is None:\n return errored, missing_meta + 1\n\n labels = meta.get(\"labels\") or {}\n if labels.get(\"complexity\"):\n buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n if labels.get(\"environment\"):\n buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n if labels.get(\"scenario\"):\n sc = labels[\"scenario\"]\n buckets[\"by_scenario\"][sc].append(score)\n buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n modality = labels.get(\"modality\") or {}\n if modality.get(\"type\"):\n buckets[\"by_modality\"][modality[\"type\"]].append(score)\n for ch in modality.get(\"channels\") or []:\n buckets[\"by_channel\"][ch].append(score)\n for cap in labels.get(\"capabilities\") or []:\n buckets[\"by_capability\"][cap].append(score)\n if meta.get(\"source_dataset\"):\n buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n if meta.get(\"category\"):\n buckets[\"by_category\"][meta[\"category\"]].append(score)\n if meta.get(\"subcategory\"):\n buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n # use the *task's* original grading_type (from md), not the run-time\n # grading_type which may be ``error``\n orig_gt = meta.get(\"grading_type\")\n if orig_gt:\n buckets[\"by_grading\"][orig_gt].append(score)\n\n return errored, missing_meta\n\n\ndef _build_task_name_index(\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n \"\"\"Map normalised task display name → t_id for fallback lookups.\n\n Used for the hermes single-summary format whose ``task_id`` field uses\n internal hermes slugs that don't match our ``t_id`` system. ``task_name``\n matches our ``name`` field exactly.\n \"\"\"\n index: dict[str, str] = {}\n for t_id, meta in task_meta.items():\n name = (meta.get(\"name\") or \"\").strip().lower()\n if name:\n index[name] = t_id\n return index\n\n\ndef _build_task_slug_index(\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n \"\"\"Map hermes ``task_id`` slugs (``core_id`` / ``task_id``) → ``t_id``.\"\"\"\n index: dict[str, str] = {}\n for t_id, meta in task_meta.items():\n index[t_id.lower()] = t_id\n for key in (\"core_id\", \"task_id\"):\n slug = (meta.get(key) or \"\").strip().lower()\n if slug:\n index[slug] = t_id\n fn = (meta.get(\"filename\") or \"\").strip()\n if fn.endswith(\".md\"):\n _prefix, sep, tail = fn[:-3].partition(\"_\")\n if sep and tail:\n index[tail.lower()] = t_id\n return index\n\n\ndef _resolve_t_id_from_hermes_result(\n r: dict[str, Any],\n name_to_tid: dict[str, str],\n slug_to_tid: dict[str, str],\n) -> str | None:\n name_key = (r.get(\"task_name\") or \"\").strip().lower()\n if name_key and name_key in name_to_tid:\n return name_to_tid[name_key]\n slug = (r.get(\"task_id\") or \"\").strip().lower()\n if slug and slug in slug_to_tid:\n return slug_to_tid[slug]\n # Some hermes exports use ``T042``-style ids in ``task_id``.\n if slug and slug.upper() in slug_to_tid:\n return slug_to_tid[slug.upper()]\n return None\n\n\ndef _is_pawbench_metrics(doc: dict[str, Any]) -> bool:\n \"\"\"True when ``metrics.json`` follows the standard per-task layout.\"\"\"\n return isinstance(doc, dict) and \"task_score\" in doc\n\n\ndef _find_hermes_summary(harness_dir: Path) -> Path | None:\n \"\"\"Pick the hermes roll-up JSON under ``<harness>/`` (not ``workspaces/``).\"\"\"\n best: tuple[int, Path] | None = None\n for sf in sorted(harness_dir.glob(\"*.json\")):\n try:\n head = json.loads(sf.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError:\n continue\n if not isinstance(head, dict):\n continue\n results = head.get(\"results\")\n if not isinstance(results, list) or not results:\n continue\n n = len(results)\n if best is None or n > best[0]:\n best = (n, sf)\n return best[1] if best else None\n\n\ndef _discover_per_task_metrics(harness_dir: Path) -> list[Path]:\n \"\"\"``T*/output/metrics.json`` files that contain pawbench ``task_score``.\"\"\"\n out: list[Path] = []\n for mp in sorted(harness_dir.glob(\"T*/output/metrics.json\")):\n try:\n doc = json.loads(mp.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError:\n continue\n if _is_pawbench_metrics(doc):\n out.append(mp)\n return out\n\n\ndef _aggregate_from_per_task_dirs(\n metrics_files: list[Path],\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n errored = 0\n missing_meta = 0\n for mp in metrics_files:\n try:\n d = json.loads(mp.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError as exc:\n print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n continue\n\n # Use the directory name as the canonical t-id source.\n t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n de, dm = _record_task(\n t_id=t_id,\n score=float(d.get(\"task_score\") or 0.0),\n grading_type=d.get(\"grading_type\") or \"unknown\",\n breakdown=d.get(\"breakdown\") or {},\n status=d.get(\"status\") or \"\",\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n errored += de\n missing_meta += dm\n return errored, missing_meta\n\n\ndef _aggregate_from_single_summary(\n summary_file: Path,\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n \"\"\"Parse hermes' single-file summary format.\n\n Schema (one JSON per ``<harness_dir>/<timestamp>.json``)::\n\n {\n \"summary\": {...},\n \"results\": [\n {\"task_id\": \"...\", \"task_name\": \"...\",\n \"score\": float, \"max_score\": float, \"passed\": bool,\n \"grading_type\": \"hybrid|automated|llm_judge|error\",\n \"breakdown\": {...}, \"status\": \"success|error|timeout\",\n \"timed_out\": bool, \"labels\": {...}},\n ...\n ]\n }\n\n The ``task_id`` field uses internal hermes slugs that don't match our\n ``t_id`` system, so we map back via ``task_name`` against ``tasks.json``.\n \"\"\"\n try:\n d = json.loads(summary_file.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError as exc:\n print(f\"[aggregate_results] skip {summary_file}: {exc}\", file=sys.stderr)\n return 0, 0\n\n results = d.get(\"results\") or []\n if not results:\n return 0, 0\n\n name_to_tid = _build_task_name_index(task_meta)\n slug_to_tid = _build_task_slug_index(task_meta)\n errored = 0\n missing_meta = 0\n for r in results:\n # ``score`` is the canonical 0-1 task score for hermes; fall back to\n # ``task_score`` for safety in case the schema evolves.\n score = float(r.get(\"score\") if r.get(\"score\") is not None else r.get(\"task_score\") or 0.0)\n status = r.get(\"status\") or \"\"\n if r.get(\"timed_out\") and status not in {\"error\", \"timeout\"}:\n # treat timeouts as errors for accounting parity with the\n # per-task-dir format\n status = \"error\"\n\n t_id = _resolve_t_id_from_hermes_result(r, name_to_tid, slug_to_tid)\n\n de, dm = _record_task(\n t_id=t_id,\n score=score,\n grading_type=r.get(\"grading_type\") or \"unknown\",\n breakdown=r.get(\"breakdown\") or {},\n status=status,\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n errored += de\n missing_meta += dm\n return errored, missing_meta\n\n\ndef aggregate_pair(\n run_dir: Path,\n model_dir: Path,\n harness_dir: Path,\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n \"\"\"Aggregate a single ``(run, model, harness)`` triple.\n\n Tries two on-disk layouts:\n\n 1. Per-task directories with ``T*/output/metrics.json`` (the default\n hermes/openclaw/qwenpaw layout).\n 2. A single ``<timestamp>.json`` summary file directly under the harness\n directory (used by some hermes runs, e.g.\n ``result/.../openai.claude-opus-4-6/hermes/20260525_115007.json``).\n \"\"\"\n per_task_score: list[float] = []\n automated_scores: list[float] = []\n judge_scores: list[float] = []\n\n # bucket → list of task scores (errors counted as 0)\n buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n metrics_files = _discover_per_task_metrics(harness_dir)\n summary_file = _find_hermes_summary(harness_dir)\n\n if metrics_files:\n errored, missing_meta = _aggregate_from_per_task_dirs(\n metrics_files,\n task_meta,\n per_task_score,\n automated_scores,\n judge_scores,\n buckets,\n )\n elif summary_file is not None:\n # Hermes roll-up export (e.g. ``openai.claude-opus-4-6/hermes/<ts>.json``)\n # with ``results[]`` instead of per-task ``T*/output/metrics.json``.\n errored, missing_meta = _aggregate_from_single_summary(\n summary_file,\n task_meta,\n per_task_score,\n automated_scores,\n judge_scores,\n buckets,\n )\n else:\n return None\n\n if not per_task_score:\n return None\n\n # Fill missing tasks with score=0 so the denominator is always the full\n # task set, preventing harness crashes from inflating the mean.\n tasks_recorded = len(per_task_score)\n total_expected = len(task_meta)\n missing_count = max(0, total_expected - tasks_recorded)\n\n # For per-task dirs we can identify exactly which tasks are missing.\n missing_tids: set[str] = set()\n if metrics_files and missing_count > 0:\n seen_tids = {mp.parent.parent.name.split(\"_\", 1)[0] for mp in metrics_files}\n missing_tids = set(task_meta.keys()) - seen_tids\n elif not metrics_files and missing_count > 0:\n # Summary format: task_name resolution can have collisions (duplicate\n # names across zh/en variants), so fall back to count-based fill.\n missing_tids = set(f\"_missing_{i}\" for i in range(missing_count))\n for t_id in missing_tids:\n _record_task(\n t_id=t_id,\n score=0.0,\n grading_type=\"missing\",\n breakdown={},\n status=\"missing\",\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n\n # Compute slice means\n by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n return {\n \"run\": run_dir.name,\n \"model\": display_model(model_dir.name),\n \"harness\": canonical_harness(harness_dir.name),\n \"overall\": safe_mean(per_task_score),\n \"automated\": safe_mean(automated_scores),\n \"judge\": safe_mean(judge_scores),\n \"tasks\": len(per_task_score),\n \"tasks_total\": len(task_meta) or len(per_task_score),\n \"tasks_errored\": errored,\n \"tasks_missing\": len(missing_tids),\n \"missing_metadata\": missing_meta,\n \"_raw_model_dir\": model_dir.name,\n \"_raw_harness_dir\": harness_dir.name,\n \"updated\": run_updated(run_dir.name),\n **by_dims,\n }\n\n\ndef main() -> int:\n ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n ap.add_argument(\n \"--run\",\n action=\"append\",\n default=None,\n help=\"Restrict to specific run directory name(s); default = all under result/\",\n )\n ap.add_argument(\n \"--out-dir\",\n default=str(SUBMISSIONS_DIR),\n help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n )\n args = ap.parse_args()\n\n runs = discover_runs(args.run)\n if not runs:\n print(\n f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n file=sys.stderr,\n )\n return 0\n\n task_meta = load_task_metadata()\n out_dir = Path(args.out_dir)\n out_dir.mkdir(parents=True, exist_ok=True)\n\n written = 0\n for run_dir in runs:\n for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n if not row:\n continue\n # Use the canonical harness slug in the filename so that\n # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n # automatically retires the old file once the user re-runs\n # the aggregator and removes the stale submissions.\n fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n out_path = out_dir / fname\n # don't ship the internal raw-name keys into the public file\n public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n out_path.write_text(\n json.dumps(public, ensure_ascii=False, indent=2),\n encoding=\"utf-8\",\n )\n written += 1\n print(\n f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s} \"\n f\"overall={row['overall']:.3f} \"\n f\"errored={row['tasks_errored']:>3d} \"\n f\"→ {out_path.relative_to(REPO_ROOT)}\"\n )\n\n print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n"}, "5d8815ec66b03301b49accbb2d81ff20e5d52a53": {"t": 1785770709.352257, "n": 20581, "h": "4ae343513b903005", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n python3 site/scripts/aggregate_results.py\n python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n # we know the official display name; users can edit this mapping.\n \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n if raw in MODEL_ALIAS:\n return MODEL_ALIAS[raw]\n head, sep, tail = raw.partition(\".\")\n if sep and head in SAFE_VENDOR_PREFIXES and tail:\n return tail\n return raw\n\n\ndef canonical_harness(raw: str) -> str:\n return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n m = RUN_DATE_RE.search(run_name)\n if m:\n y, mo, d = m.groups()\n return f\"{y}-{mo}-{d}\"\n return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n if not TASKS_JSON.exists():\n print(\n f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n \"run build_tasks.py first.\",\n file=sys.stderr,\n )\n sys.exit(2)\n tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n For purely-automated or purely-judge breakdowns the keys are unprefixed and\n we leave the partition to the caller (see ``score_partition``).\n \"\"\"\n auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n a = mean(auto_vals) if auto_vals else None\n j = mean(judge_vals) if judge_vals else None\n return a, j\n\n\ndef score_partition(\n grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n Tasks that don't contribute to a side return ``None`` for that side, which\n excludes them from the corresponding column mean.\n \"\"\"\n if grading_type == \"hybrid\":\n return split_breakdown(breakdown)\n if grading_type == \"automated\":\n return task_score, None\n if grading_type == \"llm_judge\":\n return None, task_score\n # error / unknown\n return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n if not RESULT_ROOT.is_dir():\n return []\n runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n if only:\n wanted = set(only)\n runs = [r for r in runs if r.name in wanted]\n return runs\n\n\ndef _record_task(\n *,\n t_id: str | None,\n score: float,\n grading_type: str,\n breakdown: dict[str, float],\n status: str,\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n \"\"\"Apply a single task result to the running aggregates.\n\n Returns ``(errored, missing_meta)`` deltas (0 or 1 each) so the caller\n can keep cumulative counters.\n \"\"\"\n errored = 0\n missing_meta = 0\n\n if grading_type in (\"error\", \"missing\") or status in (\"error\", \"missing\"):\n errored = 1\n score = 0.0 # ensure error/missing tasks count as 0 toward all means\n else:\n a, j = score_partition(grading_type, score, breakdown)\n if a is not None:\n automated_scores.append(a)\n if j is not None:\n judge_scores.append(j)\n\n per_task_score.append(score)\n\n meta = task_meta.get(t_id) if t_id else None\n if meta is None:\n return errored, missing_meta + 1\n\n labels = meta.get(\"labels\") or {}\n if labels.get(\"complexity\"):\n buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n if labels.get(\"environment\"):\n buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n if labels.get(\"scenario\"):\n sc = labels[\"scenario\"]\n buckets[\"by_scenario\"][sc].append(score)\n buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n modality = labels.get(\"modality\") or {}\n if modality.get(\"type\"):\n buckets[\"by_modality\"][modality[\"type\"]].append(score)\n for ch in modality.get(\"channels\") or []:\n buckets[\"by_channel\"][ch].append(score)\n for cap in labels.get(\"capabilities\") or []:\n buckets[\"by_capability\"][cap].append(score)\n if meta.get(\"source_dataset\"):\n buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n if meta.get(\"category\"):\n buckets[\"by_category\"][meta[\"category\"]].append(score)\n if meta.get(\"subcategory\"):\n buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n # use the *task's* original grading_type (from md), not the run-time\n # grading_type which may be ``error``\n orig_gt = meta.get(\"grading_type\")\n if orig_gt:\n buckets[\"by_grading\"][orig_gt].append(score)\n\n return errored, missing_meta\n\n\ndef _build_task_name_index(\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n \"\"\"Map normalised task display name → t_id for fallback lookups.\n\n Used for the hermes single-summary format whose ``task_id`` field uses\n internal hermes slugs that don't match our ``t_id`` system. ``task_name``\n matches our ``name`` field exactly.\n \"\"\"\n index: dict[str, str] = {}\n for t_id, meta in task_meta.items():\n name = (meta.get(\"name\") or \"\").strip().lower()\n if name:\n index[name] = t_id\n return index\n\n\ndef _build_task_slug_index(\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n \"\"\"Map hermes ``task_id`` slugs (``core_id`` / ``task_id``) → ``t_id``.\"\"\"\n index: dict[str, str] = {}\n for t_id, meta in task_meta.items():\n index[t_id.lower()] = t_id\n for key in (\"core_id\", \"task_id\"):\n slug = (meta.get(key) or \"\").strip().lower()\n if slug:\n index[slug] = t_id\n fn = (meta.get(\"filename\") or \"\").strip()\n if fn.endswith(\".md\"):\n _prefix, sep, tail = fn[:-3].partition(\"_\")\n if sep and tail:\n index[tail.lower()] = t_id\n return index\n\n\ndef _resolve_t_id_from_hermes_result(\n r: dict[str, Any],\n name_to_tid: dict[str, str],\n slug_to_tid: dict[str, str],\n) -> str | None:\n name_key = (r.get(\"task_name\") or \"\").strip().lower()\n if name_key and name_key in name_to_tid:\n return name_to_tid[name_key]\n slug = (r.get(\"task_id\") or \"\").strip().lower()\n if slug and slug in slug_to_tid:\n return slug_to_tid[slug]\n # Some hermes exports use ``T042``-style ids in ``task_id``.\n if slug and slug.upper() in slug_to_tid:\n return slug_to_tid[slug.upper()]\n return None\n\n\ndef _is_pawbench_metrics(doc: dict[str, Any]) -> bool:\n \"\"\"True when ``metrics.json`` follows the standard per-task layout.\"\"\"\n return isinstance(doc, dict) and \"task_score\" in doc\n\n\ndef _find_hermes_summary(harness_dir: Path) -> Path | None:\n \"\"\"Pick the hermes roll-up JSON under ``<harness>/`` (not ``workspaces/``).\"\"\"\n best: tuple[int, Path] | None = None\n for sf in sorted(harness_dir.glob(\"*.json\")):\n try:\n head = json.loads(sf.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError:\n continue\n if not isinstance(head, dict):\n continue\n results = head.get(\"results\")\n if not isinstance(results, list) or not results:\n continue\n n = len(results)\n if best is None or n > best[0]:\n best = (n, sf)\n return best[1] if best else None\n\n\ndef _discover_per_task_metrics(harness_dir: Path) -> list[Path]:\n \"\"\"``T*/output/metrics.json`` files that contain pawbench ``task_score``.\"\"\"\n out: list[Path] = []\n for mp in sorted(harness_dir.glob(\"T*/output/metrics.json\")):\n try:\n doc = json.loads(mp.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError:\n continue\n if _is_pawbench_metrics(doc):\n out.append(mp)\n return out\n\n\ndef _aggregate_from_per_task_dirs(\n metrics_files: list[Path],\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n errored = 0\n missing_meta = 0\n for mp in metrics_files:\n try:\n d = json.loads(mp.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError as exc:\n print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n continue\n\n # Use the directory name as the canonical t-id source.\n t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n de, dm = _record_task(\n t_id=t_id,\n score=float(d.get(\"task_score\") or 0.0),\n grading_type=d.get(\"grading_type\") or \"unknown\",\n breakdown=d.get(\"breakdown\") or {},\n status=d.get(\"status\") or \"\",\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n errored += de\n missing_meta += dm\n return errored, missing_meta\n\n\ndef _aggregate_from_single_summary(\n summary_file: Path,\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n \"\"\"Parse hermes' single-file summary format.\n\n Schema (one JSON per ``<harness_dir>/<timestamp>.json``)::\n\n {\n \"summary\": {...},\n \"results\": [\n {\"task_id\": \"...\", \"task_name\": \"...\",\n \"score\": float, \"max_score\": float, \"passed\": bool,\n \"grading_type\": \"hybrid|automated|llm_judge|error\",\n \"breakdown\": {...}, \"status\": \"success|error|timeout\",\n \"timed_out\": bool, \"labels\": {...}},\n ...\n ]\n }\n\n The ``task_id`` field uses internal hermes slugs that don't match our\n ``t_id`` system, so we map back via ``task_name`` against ``tasks.json``.\n \"\"\"\n try:\n d = json.loads(summary_file.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError as exc:\n print(f\"[aggregate_results] skip {summary_file}: {exc}\", file=sys.stderr)\n return 0, 0\n\n results = d.get(\"results\") or []\n if not results:\n return 0, 0\n\n name_to_tid = _build_task_name_index(task_meta)\n slug_to_tid = _build_task_slug_index(task_meta)\n errored = 0\n missing_meta = 0\n for r in results:\n # ``score`` is the canonical 0-1 task score for hermes; fall back to\n # ``task_score`` for safety in case the schema evolves.\n score = float(r.get(\"score\") if r.get(\"score\") is not None else r.get(\"task_score\") or 0.0)\n status = r.get(\"status\") or \"\"\n if r.get(\"timed_out\") and status not in {\"error\", \"timeout\"}:\n # treat timeouts as errors for accounting parity with the\n # per-task-dir format\n status = \"error\"\n\n t_id = _resolve_t_id_from_hermes_result(r, name_to_tid, slug_to_tid)\n\n de, dm = _record_task(\n t_id=t_id,\n score=score,\n grading_type=r.get(\"grading_type\") or \"unknown\",\n breakdown=r.get(\"breakdown\") or {},\n status=status,\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n errored += de\n missing_meta += dm\n return errored, missing_meta\n\n\ndef aggregate_pair(\n run_dir: Path,\n model_dir: Path,\n harness_dir: Path,\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n \"\"\"Aggregate a single ``(run, model, harness)`` triple.\n\n Tries two on-disk layouts:\n\n 1. Per-task directories with ``T*/output/metrics.json`` (the default\n hermes/openclaw/qwenpaw layout).\n 2. A single ``<timestamp>.json`` summary file directly under the harness\n directory (used by some hermes runs, e.g.\n ``result/.../openai.claude-opus-4-6/hermes/20260525_115007.json``).\n \"\"\"\n per_task_score: list[float] = []\n automated_scores: list[float] = []\n judge_scores: list[float] = []\n\n # bucket → list of task scores (errors counted as 0)\n buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n metrics_files = _discover_per_task_metrics(harness_dir)\n summary_file = _find_hermes_summary(harness_dir)\n\n if metrics_files:\n errored, missing_meta = _aggregate_from_per_task_dirs(\n metrics_files,\n task_meta,\n per_task_score,\n automated_scores,\n judge_scores,\n buckets,\n )\n elif summary_file is not None:\n # Hermes roll-up export (e.g. ``openai.claude-opus-4-6/hermes/<ts>.json``)\n # with ``results[]`` instead of per-task ``T*/output/metrics.json``.\n errored, missing_meta = _aggregate_from_single_summary(\n summary_file,\n task_meta,\n per_task_score,\n automated_scores,\n judge_scores,\n buckets,\n )\n else:\n return None\n\n if not per_task_score:\n return None\n\n # Fill missing tasks with score=0 so the denominator is always the full\n # task set, preventing harness crashes from inflating the mean.\n tasks_recorded = len(per_task_score)\n total_expected = len(task_meta)\n missing_count = max(0, total_expected - tasks_recorded)\n\n # For per-task dirs we can identify exactly which tasks are missing.\n missing_tids: set[str] = set()\n if metrics_files and missing_count > 0:\n seen_tids = {mp.parent.parent.name.split(\"_\", 1)[0] for mp in metrics_files}\n missing_tids = set(task_meta.keys()) - seen_tids\n elif not metrics_files and missing_count > 0:\n # Summary format: task_name resolution can have collisions (duplicate\n # names across zh/en variants), so fall back to count-based fill.\n missing_tids = set(f\"_missing_{i}\" for i in range(missing_count))\n for t_id in missing_tids:\n _record_task(\n t_id=t_id,\n score=0.0,\n grading_type=\"missing\",\n breakdown={},\n status=\"missing\",\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n\n # Compute slice means\n by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n return {\n \"run\": run_dir.name,\n \"model\": display_model(model_dir.name),\n \"harness\": canonical_harness(harness_dir.name),\n \"overall\": safe_mean(per_task_score),\n \"automated\": safe_mean(automated_scores),\n \"judge\": safe_mean(judge_scores),\n \"tasks\": len(per_task_score),\n \"tasks_total\": len(task_meta) or len(per_task_score),\n \"tasks_errored\": errored,\n \"tasks_missing\": len(missing_tids),\n \"missing_metadata\": missing_meta,\n \"_raw_model_dir\": model_dir.name,\n \"_raw_harness_dir\": harness_dir.name,\n \"updated\": run_updated(run_dir.name),\n **by_dims,\n }\n\n\ndef main() -> int:\n ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n ap.add_argument(\n \"--run\",\n action=\"append\",\n default=None,\n help=\"Restrict to specific run directory name(s); default = all under result/\",\n )\n ap.add_argument(\n \"--out-dir\",\n default=str(SUBMISSIONS_DIR),\n help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n )\n args = ap.parse_args()\n\n runs = discover_runs(args.run)\n if not runs:\n print(\n f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n file=sys.stderr,\n )\n return 0\n\n task_meta = load_task_metadata()\n out_dir = Path(args.out_dir)\n out_dir.mkdir(parents=True, exist_ok=True)\n\n written = 0\n for run_dir in runs:\n for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n if not row:\n continue\n # Use the canonical harness slug in the filename so that\n # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n # automatically retires the old file once the user re-runs\n # the aggregator and removes the stale submissions.\n fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n out_path = out_dir / fname\n # don't ship the internal raw-name keys into the public file\n public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n out_path.write_text(\n json.dumps(public, ensure_ascii=False, indent=2),\n encoding=\"utf-8\",\n )\n written += 1\n print(\n f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s} \"\n f\"overall={row['overall']:.3f} \"\n f\"errored={row['tasks_errored']:>3d} \"\n f\"→ {out_path.relative_to(REPO_ROOT)}\"\n )\n\n print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n"}, "c9dc00a821f4c89b3cadc6ec436fc81e5a3d1f2e": {"t": 1785770710.666815, "n": 19259, "h": "c96cefdd0a9e3666", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n python3 site/scripts/aggregate_results.py\n python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n # we know the official display name; users can edit this mapping.\n \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n if raw in MODEL_ALIAS:\n return MODEL_ALIAS[raw]\n head, sep, tail = raw.partition(\".\")\n if sep and head in SAFE_VENDOR_PREFIXES and tail:\n return tail\n return raw\n\n\ndef canonical_harness(raw: str) -> str:\n return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n m = RUN_DATE_RE.search(run_name)\n if m:\n y, mo, d = m.groups()\n return f\"{y}-{mo}-{d}\"\n return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n if not TASKS_JSON.exists():\n print(\n f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n \"run build_tasks.py first.\",\n file=sys.stderr,\n )\n sys.exit(2)\n tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n For purely-automated or purely-judge breakdowns the keys are unprefixed and\n we leave the partition to the caller (see ``score_partition``).\n \"\"\"\n auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n a = mean(auto_vals) if auto_vals else None\n j = mean(judge_vals) if judge_vals else None\n return a, j\n\n\ndef score_partition(\n grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n Tasks that don't contribute to a side return ``None`` for that side, which\n excludes them from the corresponding column mean.\n \"\"\"\n if grading_type == \"hybrid\":\n return split_breakdown(breakdown)\n if grading_type == \"automated\":\n return task_score, None\n if grading_type == \"llm_judge\":\n return None, task_score\n # error / unknown\n return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n if not RESULT_ROOT.is_dir():\n return []\n runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n if only:\n wanted = set(only)\n runs = [r for r in runs if r.name in wanted]\n return runs\n\n\ndef _record_task(\n *,\n t_id: str | None,\n score: float,\n grading_type: str,\n breakdown: dict[str, float],\n status: str,\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n \"\"\"Apply a single task result to the running aggregates.\n\n Returns ``(errored, missing_meta)`` deltas (0 or 1 each) so the caller\n can keep cumulative counters.\n \"\"\"\n errored = 0\n missing_meta = 0\n\n if grading_type == \"error\" or status == \"error\":\n errored = 1\n score = 0.0 # ensure error tasks count as 0 toward all means\n else:\n a, j = score_partition(grading_type, score, breakdown)\n if a is not None:\n automated_scores.append(a)\n if j is not None:\n judge_scores.append(j)\n\n per_task_score.append(score)\n\n meta = task_meta.get(t_id) if t_id else None\n if meta is None:\n return errored, missing_meta + 1\n\n labels = meta.get(\"labels\") or {}\n if labels.get(\"complexity\"):\n buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n if labels.get(\"environment\"):\n buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n if labels.get(\"scenario\"):\n sc = labels[\"scenario\"]\n buckets[\"by_scenario\"][sc].append(score)\n buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n modality = labels.get(\"modality\") or {}\n if modality.get(\"type\"):\n buckets[\"by_modality\"][modality[\"type\"]].append(score)\n for ch in modality.get(\"channels\") or []:\n buckets[\"by_channel\"][ch].append(score)\n for cap in labels.get(\"capabilities\") or []:\n buckets[\"by_capability\"][cap].append(score)\n if meta.get(\"source_dataset\"):\n buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n if meta.get(\"category\"):\n buckets[\"by_category\"][meta[\"category\"]].append(score)\n if meta.get(\"subcategory\"):\n buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n # use the *task's* original grading_type (from md), not the run-time\n # grading_type which may be ``error``\n orig_gt = meta.get(\"grading_type\")\n if orig_gt:\n buckets[\"by_grading\"][orig_gt].append(score)\n\n return errored, missing_meta\n\n\ndef _build_task_name_index(\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n \"\"\"Map normalised task display name → t_id for fallback lookups.\n\n Used for the hermes single-summary format whose ``task_id`` field uses\n internal hermes slugs that don't match our ``t_id`` system. ``task_name``\n matches our ``name`` field exactly.\n \"\"\"\n index: dict[str, str] = {}\n for t_id, meta in task_meta.items():\n name = (meta.get(\"name\") or \"\").strip().lower()\n if name:\n index[name] = t_id\n return index\n\n\ndef _build_task_slug_index(\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, str]:\n \"\"\"Map hermes ``task_id`` slugs (``core_id`` / ``task_id``) → ``t_id``.\"\"\"\n index: dict[str, str] = {}\n for t_id, meta in task_meta.items():\n index[t_id.lower()] = t_id\n for key in (\"core_id\", \"task_id\"):\n slug = (meta.get(key) or \"\").strip().lower()\n if slug:\n index[slug] = t_id\n fn = (meta.get(\"filename\") or \"\").strip()\n if fn.endswith(\".md\"):\n _prefix, sep, tail = fn[:-3].partition(\"_\")\n if sep and tail:\n index[tail.lower()] = t_id\n return index\n\n\ndef _resolve_t_id_from_hermes_result(\n r: dict[str, Any],\n name_to_tid: dict[str, str],\n slug_to_tid: dict[str, str],\n) -> str | None:\n name_key = (r.get(\"task_name\") or \"\").strip().lower()\n if name_key and name_key in name_to_tid:\n return name_to_tid[name_key]\n slug = (r.get(\"task_id\") or \"\").strip().lower()\n if slug and slug in slug_to_tid:\n return slug_to_tid[slug]\n # Some hermes exports use ``T042``-style ids in ``task_id``.\n if slug and slug.upper() in slug_to_tid:\n return slug_to_tid[slug.upper()]\n return None\n\n\ndef _is_pawbench_metrics(doc: dict[str, Any]) -> bool:\n \"\"\"True when ``metrics.json`` follows the standard per-task layout.\"\"\"\n return isinstance(doc, dict) and \"task_score\" in doc\n\n\ndef _find_hermes_summary(harness_dir: Path) -> Path | None:\n \"\"\"Pick the hermes roll-up JSON under ``<harness>/`` (not ``workspaces/``).\"\"\"\n best: tuple[int, Path] | None = None\n for sf in sorted(harness_dir.glob(\"*.json\")):\n try:\n head = json.loads(sf.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError:\n continue\n if not isinstance(head, dict):\n continue\n results = head.get(\"results\")\n if not isinstance(results, list) or not results:\n continue\n n = len(results)\n if best is None or n > best[0]:\n best = (n, sf)\n return best[1] if best else None\n\n\ndef _discover_per_task_metrics(harness_dir: Path) -> list[Path]:\n \"\"\"``T*/output/metrics.json`` files that contain pawbench ``task_score``.\"\"\"\n out: list[Path] = []\n for mp in sorted(harness_dir.glob(\"T*/output/metrics.json\")):\n try:\n doc = json.loads(mp.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError:\n continue\n if _is_pawbench_metrics(doc):\n out.append(mp)\n return out\n\n\ndef _aggregate_from_per_task_dirs(\n metrics_files: list[Path],\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n errored = 0\n missing_meta = 0\n for mp in metrics_files:\n try:\n d = json.loads(mp.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError as exc:\n print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n continue\n\n # Use the directory name as the canonical t-id source.\n t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n de, dm = _record_task(\n t_id=t_id,\n score=float(d.get(\"task_score\") or 0.0),\n grading_type=d.get(\"grading_type\") or \"unknown\",\n breakdown=d.get(\"breakdown\") or {},\n status=d.get(\"status\") or \"\",\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n errored += de\n missing_meta += dm\n return errored, missing_meta\n\n\ndef _aggregate_from_single_summary(\n summary_file: Path,\n task_meta: dict[str, dict[str, Any]],\n per_task_score: list[float],\n automated_scores: list[float],\n judge_scores: list[float],\n buckets: dict[str, dict[str, list[float]]],\n) -> tuple[int, int]:\n \"\"\"Parse hermes' single-file summary format.\n\n Schema (one JSON per ``<harness_dir>/<timestamp>.json``)::\n\n {\n \"summary\": {...},\n \"results\": [\n {\"task_id\": \"...\", \"task_name\": \"...\",\n \"score\": float, \"max_score\": float, \"passed\": bool,\n \"grading_type\": \"hybrid|automated|llm_judge|error\",\n \"breakdown\": {...}, \"status\": \"success|error|timeout\",\n \"timed_out\": bool, \"labels\": {...}},\n ...\n ]\n }\n\n The ``task_id`` field uses internal hermes slugs that don't match our\n ``t_id`` system, so we map back via ``task_name`` against ``tasks.json``.\n \"\"\"\n try:\n d = json.loads(summary_file.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError as exc:\n print(f\"[aggregate_results] skip {summary_file}: {exc}\", file=sys.stderr)\n return 0, 0\n\n results = d.get(\"results\") or []\n if not results:\n return 0, 0\n\n name_to_tid = _build_task_name_index(task_meta)\n slug_to_tid = _build_task_slug_index(task_meta)\n errored = 0\n missing_meta = 0\n for r in results:\n # ``score`` is the canonical 0-1 task score for hermes; fall back to\n # ``task_score`` for safety in case the schema evolves.\n score = float(r.get(\"score\") if r.get(\"score\") is not None else r.get(\"task_score\") or 0.0)\n status = r.get(\"status\") or \"\"\n if r.get(\"timed_out\") and status not in {\"error\", \"timeout\"}:\n # treat timeouts as errors for accounting parity with the\n # per-task-dir format\n status = \"error\"\n\n t_id = _resolve_t_id_from_hermes_result(r, name_to_tid, slug_to_tid)\n\n de, dm = _record_task(\n t_id=t_id,\n score=score,\n grading_type=r.get(\"grading_type\") or \"unknown\",\n breakdown=r.get(\"breakdown\") or {},\n status=status,\n task_meta=task_meta,\n per_task_score=per_task_score,\n automated_scores=automated_scores,\n judge_scores=judge_scores,\n buckets=buckets,\n )\n errored += de\n missing_meta += dm\n return errored, missing_meta\n\n\ndef aggregate_pair(\n run_dir: Path,\n model_dir: Path,\n harness_dir: Path,\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n \"\"\"Aggregate a single ``(run, model, harness)`` triple.\n\n Tries two on-disk layouts:\n\n 1. Per-task directories with ``T*/output/metrics.json`` (the default\n hermes/openclaw/qwenpaw layout).\n 2. A single ``<timestamp>.json`` summary file directly under the harness\n directory (used by some hermes runs, e.g.\n ``result/.../openai.claude-opus-4-6/hermes/20260525_115007.json``).\n \"\"\"\n per_task_score: list[float] = []\n automated_scores: list[float] = []\n judge_scores: list[float] = []\n\n # bucket → list of task scores (errors counted as 0)\n buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n metrics_files = _discover_per_task_metrics(harness_dir)\n summary_file = _find_hermes_summary(harness_dir)\n\n if metrics_files:\n errored, missing_meta = _aggregate_from_per_task_dirs(\n metrics_files,\n task_meta,\n per_task_score,\n automated_scores,\n judge_scores,\n buckets,\n )\n elif summary_file is not None:\n # Hermes roll-up export (e.g. ``openai.claude-opus-4-6/hermes/<ts>.json``)\n # with ``results[]`` instead of per-task ``T*/output/metrics.json``.\n errored, missing_meta = _aggregate_from_single_summary(\n summary_file,\n task_meta,\n per_task_score,\n automated_scores,\n judge_scores,\n buckets,\n )\n else:\n return None\n\n if not per_task_score:\n return None\n\n # Compute slice means\n by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n return {\n \"run\": run_dir.name,\n \"model\": display_model(model_dir.name),\n \"harness\": canonical_harness(harness_dir.name),\n \"overall\": safe_mean(per_task_score),\n \"automated\": safe_mean(automated_scores),\n \"judge\": safe_mean(judge_scores),\n \"tasks\": len(per_task_score),\n \"tasks_total\": len(task_meta) or len(per_task_score),\n \"tasks_errored\": errored,\n \"missing_metadata\": missing_meta,\n \"_raw_model_dir\": model_dir.name,\n \"_raw_harness_dir\": harness_dir.name,\n \"updated\": run_updated(run_dir.name),\n **by_dims,\n }\n\n\ndef main() -> int:\n ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n ap.add_argument(\n \"--run\",\n action=\"append\",\n default=None,\n help=\"Restrict to specific run directory name(s); default = all under result/\",\n )\n ap.add_argument(\n \"--out-dir\",\n default=str(SUBMISSIONS_DIR),\n help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n )\n args = ap.parse_args()\n\n runs = discover_runs(args.run)\n if not runs:\n print(\n f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n file=sys.stderr,\n )\n return 0\n\n task_meta = load_task_metadata()\n out_dir = Path(args.out_dir)\n out_dir.mkdir(parents=True, exist_ok=True)\n\n written = 0\n for run_dir in runs:\n for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n if not row:\n continue\n # Use the canonical harness slug in the filename so that\n # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n # automatically retires the old file once the user re-runs\n # the aggregator and removes the stale submissions.\n fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n out_path = out_dir / fname\n # don't ship the internal raw-name keys into the public file\n public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n out_path.write_text(\n json.dumps(public, ensure_ascii=False, indent=2),\n encoding=\"utf-8\",\n )\n written += 1\n print(\n f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s} \"\n f\"overall={row['overall']:.3f} \"\n f\"errored={row['tasks_errored']:>3d} \"\n f\"→ {out_path.relative_to(REPO_ROOT)}\"\n )\n\n print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n"}, "d40f2e9686710323df316efcd2fa179dd751be60": {"t": 1785770712.0188928, "n": 11663, "h": "cb87a7c3736d63ba", "c": "#!/usr/bin/env python3\n\"\"\"Aggregate raw evaluation runs under ``result/`` into submission JSONs.\n\nFor each ``result/<run>/<model>/<harness>/<task>/output/metrics.json`` we read\nthe per-task ``task_score`` and ``breakdown`` and join with the per-task\nmetadata in ``site/src/data/tasks.json`` (built by ``build_tasks.py``) to\nproduce a single rolled-up JSON per ``(run, model, harness)`` under\n``submissions/``. ``build_leaderboard.py`` then consumes those JSONs.\n\nThis script is idempotent: re-running regenerates all ``<run>__*.json`` files.\nOld submission files for retired runs can be removed manually; the aggregator\nnever deletes anything it didn't write.\n\nWhy an intermediate file?\n\n- ``submissions/`` already exists in the project as the canonical hand-off\n format, so manually-curated rows can sit alongside auto-generated ones.\n- Decoupling raw runs from the leaderboard lets us host multiple runs (e.g.\n ``pawbench-rerun-20260519`` + a future ``pawbench-rerun-20260612``) and let\n ``build_leaderboard.py`` pick the freshest ``(model, harness)``.\n\nUsage::\n\n python3 site/scripts/aggregate_results.py\n python3 site/scripts/aggregate_results.py --run pawbench-rerun-20260519\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom collections import defaultdict\nfrom datetime import date\nfrom pathlib import Path\nfrom statistics import mean\nfrom typing import Any\n\nREPO_ROOT = Path(__file__).resolve().parents[2]\nSITE_ROOT = Path(__file__).resolve().parents[1]\nRESULT_ROOT = REPO_ROOT / \"result\"\nSUBMISSIONS_DIR = REPO_ROOT / \"submissions\"\nTASKS_JSON = SITE_ROOT / \"src\" / \"data\" / \"tasks.json\"\n\n# Map raw model directory name → display name.\n# Edit here when a new run uses a vendor-prefixed or codename folder. We do\n# not auto-strip ``vendor.`` prefixes because version dots like\n# ``qwen3.6-27b`` would be misread.\nMODEL_ALIAS: dict[str, str] = {\n \"openai.claude-opus-4-6\": \"claude-opus-4.6\",\n \"deepseek.deepseek-v4-pro\": \"deepseek-v4-pro\",\n # `q37max0515s12ep` looks like an internal codename — keep verbatim until\n # we know the official display name; users can edit this mapping.\n \"q37max0515s12ep\": \"q37max0515s12ep\",\n}\n\n# Vendor prefixes that are safe to strip (\"openai.foo\" → \"foo\"). Anything\n# matching `<vendor>.<rest>` where vendor is in this set gets the prefix\n# dropped if no explicit alias was provided.\nSAFE_VENDOR_PREFIXES: set[str] = {\"openai\", \"deepseek\", \"anthropic\", \"google\", \"meta\"}\n\n# Map raw harness directory name → canonical slug used in the leaderboard.\n# Display names live in build_leaderboard.py; this map is just for storage.\nHARNESS_ALIAS: dict[str, str] = {\n \"copaw\": \"qwenpaw\",\n}\n\nRUN_DATE_RE = re.compile(r\"(\\d{4})(\\d{2})(\\d{2})\")\n\n\ndef display_model(raw: str) -> str:\n if raw in MODEL_ALIAS:\n return MODEL_ALIAS[raw]\n head, sep, tail = raw.partition(\".\")\n if sep and head in SAFE_VENDOR_PREFIXES and tail:\n return tail\n return raw\n\n\ndef canonical_harness(raw: str) -> str:\n return HARNESS_ALIAS.get(raw, raw)\n\n\ndef run_updated(run_name: str) -> str:\n m = RUN_DATE_RE.search(run_name)\n if m:\n y, mo, d = m.groups()\n return f\"{y}-{mo}-{d}\"\n return date.today().isoformat()\n\n\ndef load_task_metadata() -> dict[str, dict[str, Any]]:\n if not TASKS_JSON.exists():\n print(\n f\"[aggregate_results] {TASKS_JSON.relative_to(REPO_ROOT)} missing — \"\n \"run build_tasks.py first.\",\n file=sys.stderr,\n )\n sys.exit(2)\n tasks = json.loads(TASKS_JSON.read_text(encoding=\"utf-8\"))\n return {t[\"t_id\"]: t for t in tasks}\n\n\ndef split_breakdown(d: dict[str, float]) -> tuple[float | None, float | None]:\n \"\"\"Pull ``automated.*`` / ``llm_judge.*`` means out of a hybrid breakdown.\n\n For purely-automated or purely-judge breakdowns the keys are unprefixed and\n we leave the partition to the caller (see ``score_partition``).\n \"\"\"\n auto_vals = [v for k, v in d.items() if k.startswith(\"automated.\")]\n judge_vals = [v for k, v in d.items() if k.startswith(\"llm_judge.\")]\n a = mean(auto_vals) if auto_vals else None\n j = mean(judge_vals) if judge_vals else None\n return a, j\n\n\ndef score_partition(\n grading_type: str, task_score: float, breakdown: dict[str, float]\n) -> tuple[float | None, float | None]:\n \"\"\"Return ``(automated_score, judge_score)`` for a single task.\n\n Tasks that don't contribute to a side return ``None`` for that side, which\n excludes them from the corresponding column mean.\n \"\"\"\n if grading_type == \"hybrid\":\n return split_breakdown(breakdown)\n if grading_type == \"automated\":\n return task_score, None\n if grading_type == \"llm_judge\":\n return None, task_score\n # error / unknown\n return None, None\n\n\ndef safe_mean(xs: list[float]) -> float:\n return round(mean(xs), 4) if xs else 0.0\n\n\ndef safe_mean_or_none(xs: list[float]) -> float | None:\n return round(mean(xs), 4) if xs else None\n\n\ndef discover_runs(only: list[str] | None) -> list[Path]:\n if not RESULT_ROOT.is_dir():\n return []\n runs = sorted(p for p in RESULT_ROOT.iterdir() if p.is_dir() and not p.name.startswith(\".\"))\n if only:\n wanted = set(only)\n runs = [r for r in runs if r.name in wanted]\n return runs\n\n\ndef aggregate_pair(\n run_dir: Path,\n model_dir: Path,\n harness_dir: Path,\n task_meta: dict[str, dict[str, Any]],\n) -> dict[str, Any] | None:\n \"\"\"Aggregate a single ``(run, model, harness)`` triple.\"\"\"\n metrics_files = sorted(harness_dir.glob(\"T*/output/metrics.json\"))\n if not metrics_files:\n return None\n\n per_task_score: list[float] = []\n automated_scores: list[float] = []\n judge_scores: list[float] = []\n errored = 0\n missing_meta = 0\n\n # bucket → list of task scores (errors counted as 0)\n buckets: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))\n\n seen_ids: set[str] = set()\n for mp in metrics_files:\n try:\n d = json.loads(mp.read_text(encoding=\"utf-8\"))\n except json.JSONDecodeError as exc:\n print(f\"[aggregate_results] skip {mp}: {exc}\", file=sys.stderr)\n continue\n\n # Use the directory name as the canonical t-id source; fall back to the\n # ``instance_id`` or the dir prefix when missing.\n t_id = mp.parent.parent.name.split(\"_\", 1)[0]\n seen_ids.add(t_id)\n\n score = float(d.get(\"task_score\") or 0.0)\n gt = d.get(\"grading_type\") or \"unknown\"\n breakdown = d.get(\"breakdown\") or {}\n\n if gt == \"error\" or d.get(\"status\") == \"error\":\n errored += 1\n score = 0.0 # ensure error tasks count as 0 toward all means\n else:\n a, j = score_partition(gt, score, breakdown)\n if a is not None:\n automated_scores.append(a)\n if j is not None:\n judge_scores.append(j)\n\n per_task_score.append(score)\n\n meta = task_meta.get(t_id)\n if meta is None:\n missing_meta += 1\n continue\n\n labels = meta.get(\"labels\") or {}\n # single-valued buckets\n if labels.get(\"complexity\"):\n buckets[\"by_complexity\"][labels[\"complexity\"]].append(score)\n if labels.get(\"environment\"):\n buckets[\"by_environment\"][labels[\"environment\"]].append(score)\n if labels.get(\"scenario\"):\n sc = labels[\"scenario\"]\n buckets[\"by_scenario\"][sc].append(score)\n buckets[\"by_scenario_top\"][sc.split(\"/\", 1)[0]].append(score)\n modality = labels.get(\"modality\") or {}\n if modality.get(\"type\"):\n buckets[\"by_modality\"][modality[\"type\"]].append(score)\n for ch in modality.get(\"channels\") or []:\n buckets[\"by_channel\"][ch].append(score)\n for cap in labels.get(\"capabilities\") or []:\n buckets[\"by_capability\"][cap].append(score)\n if meta.get(\"source_dataset\"):\n buckets[\"by_source\"][meta[\"source_dataset\"]].append(score)\n if meta.get(\"category\"):\n buckets[\"by_category\"][meta[\"category\"]].append(score)\n if meta.get(\"subcategory\"):\n buckets[\"by_subcategory\"][meta[\"subcategory\"]].append(score)\n # use the *task's* original grading_type (from md), not the run-time\n # grading_type which may be ``error``\n orig_gt = meta.get(\"grading_type\")\n if orig_gt:\n buckets[\"by_grading\"][orig_gt].append(score)\n\n if not per_task_score:\n return None\n\n # Compute slice means\n by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}\n\n return {\n \"run\": run_dir.name,\n \"model\": display_model(model_dir.name),\n \"harness\": canonical_harness(harness_dir.name),\n \"overall\": safe_mean(per_task_score),\n \"automated\": safe_mean(automated_scores),\n \"judge\": safe_mean(judge_scores),\n \"tasks\": len(per_task_score),\n \"tasks_total\": len(task_meta) or len(per_task_score),\n \"tasks_errored\": errored,\n \"missing_metadata\": missing_meta,\n \"_raw_model_dir\": model_dir.name,\n \"_raw_harness_dir\": harness_dir.name,\n \"updated\": run_updated(run_dir.name),\n **by_dims,\n }\n\n\ndef main() -> int:\n ap = argparse.ArgumentParser(description=__doc__.split(\"\\n\", 1)[0])\n ap.add_argument(\n \"--run\",\n action=\"append\",\n default=None,\n help=\"Restrict to specific run directory name(s); default = all under result/\",\n )\n ap.add_argument(\n \"--out-dir\",\n default=str(SUBMISSIONS_DIR),\n help=\"Where to write submission JSONs (default: <repo>/submissions/)\",\n )\n args = ap.parse_args()\n\n runs = discover_runs(args.run)\n if not runs:\n print(\n f\"[aggregate_results] no runs found under {RESULT_ROOT.relative_to(REPO_ROOT)}\",\n file=sys.stderr,\n )\n return 0\n\n task_meta = load_task_metadata()\n out_dir = Path(args.out_dir)\n out_dir.mkdir(parents=True, exist_ok=True)\n\n written = 0\n for run_dir in runs:\n for model_dir in sorted(p for p in run_dir.iterdir() if p.is_dir()):\n for harness_dir in sorted(p for p in model_dir.iterdir() if p.is_dir()):\n row = aggregate_pair(run_dir, model_dir, harness_dir, task_meta)\n if not row:\n continue\n # Use the canonical harness slug in the filename so that\n # renaming a harness alias (e.g. ``copaw`` → ``qwenpaw``)\n # automatically retires the old file once the user re-runs\n # the aggregator and removes the stale submissions.\n fname = f\"{run_dir.name}__{row['_raw_model_dir']}__{row['harness']}.json\"\n out_path = out_dir / fname\n # don't ship the internal raw-name keys into the public file\n public = {k: v for k, v in row.items() if not k.startswith(\"_\")}\n out_path.write_text(\n json.dumps(public, ensure_ascii=False, indent=2),\n encoding=\"utf-8\",\n )\n written += 1\n print(\n f\"[aggregate_results] {row['model']:30s} × {row['harness']:10s} \"\n f\"overall={row['overall']:.3f} \"\n f\"errored={row['tasks_errored']:>3d} \"\n f\"→ {out_path.relative_to(REPO_ROOT)}\"\n )\n\n print(f\"[aggregate_results] wrote {written} submission file(s) to {out_dir}\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n"}}}} |