OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
15 KiB
JSON
1 line
15 KiB
JSON
{"repo": "bioagent-bench/bioagent-experiments", "files": {"plotting/heatmap_bar.py": {"f1e489bbdd7abba8334eb5dd8f01d70cba13e328": {"t": 1786068553.1289797, "n": 14084, "h": "a4696b53021bed0b", "c": "import argparse\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom matplotlib import font_manager\nfrom matplotlib.colors import LinearSegmentedColormap, to_rgb\nfrom matplotlib.offsetbox import OffsetImage, AnnotationBbox\n\n\nASSETS_DIR = Path(__file__).resolve().parent / \"assets\"\nLOGO_DIR = ASSETS_DIR / \"logos\"\nTASK_ICON_DIR = ASSETS_DIR / \"tasks\"\nFONT_DIR = ASSETS_DIR / \"fonts\"\n# Static Figtree instances by weight (matplotlib only reads the variable\n# font's Light default, so we ship pre-instanced weights).\nFONT_WEIGHTS = {\n \"medium\": FONT_DIR / \"Figtree-Medium.ttf\",\n \"semibold\": FONT_DIR / \"Figtree-SemiBold.ttf\",\n}\nFONT_FALLBACK = FONT_DIR / \"Figtree.ttf\"\n\n# Color anchoring a 100% completion rate; the cell gradient runs white -> this.\nFULL_RATE_COLOR = \"#13682A\"\n\n# Font sizes, shared with plotting/scatter_plan_completion.py so the two\n# figures are typographically consistent. TICK_FONTSIZE is the value used for\n# the 10-model/10-task paper figures; smaller grids bump up (see below).\nAXIS_FONTSIZE = 29\nTICK_FONTSIZE = 23\n\n\nDEFAULT_MODELS = [\n 'gpt-5-1-codex-max',\n 'gpt-5-2',\n 'gpt-gemini',\n 'gpt-opus',\n 'gpt-sonnet',\n 'openrouter/mistralai/devstral-2512:free',\n 'openrouter/moonshotai/kimi-k2-thinking',\n 'openrouter/qwen/qwen3-coder:free',\n 'openrouter/z-ai/glm-4.7',\n 'openrouter/minimax/minimax-m2.1',\n]\n\n# Human-facing model names (real product names, not run identifiers).\nMODEL_LABELS = {\n 'gpt-5-1-codex-max': 'GPT-5.1',\n 'gpt-5-2': 'GPT-5.2',\n 'gpt-gemini': 'Gemini 3 Pro',\n 'gpt-opus': 'Claude Opus 4.5',\n 'gpt-sonnet': 'Claude Sonnet 4.5',\n 'openrouter/mistralai/devstral-2512:free': 'Devstral 2512',\n 'openrouter/moonshotai/kimi-k2-thinking': 'Kimi K2 Thinking',\n 'openrouter/qwen/qwen3-coder:free': 'Qwen3 Coder',\n 'openrouter/z-ai/glm-4.7': 'GLM-4.7',\n 'openrouter/minimax/minimax-m2.1': 'MiniMax M2.1',\n 'openrouter/openai/gpt-5.5': 'GPT-5.5',\n 'openrouter/openai/gpt-5.2': 'GPT-5.2',\n 'openrouter/openai/gpt-5.1': 'GPT-5.1',\n 'openrouter/moonshotai/kimi-k2.6': 'Kimi K2.6',\n 'openrouter/anthropic/claude-opus-4.7': 'Claude Opus 4.7',\n 'openrouter/deepseek/deepseek-v4-pro': 'DeepSeek V4 Pro',\n 'openrouter/qwen/qwen3.7-max': 'Qwen3.7 Max',\n 'openrouter/qwen/qwen3.6-27b': 'Qwen3.6 27B',\n 'openrouter/anthropic/claude-opus-4.5': 'Claude Opus 4.5',\n 'openrouter/google/gemini-3-pro-preview': 'Gemini 3 Pro',\n}\n\n\ndef setup_font(weight: str = \"medium\") -> str:\n \"\"\"Register the bundled Figtree font and make it the default family.\n\n ``weight`` selects a pre-instanced static weight (\"medium\" or \"semibold\").\n \"\"\"\n path = FONT_WEIGHTS.get(weight, FONT_WEIGHTS[\"medium\"])\n if not path.exists():\n path = FONT_FALLBACK\n if path.exists():\n font_manager.fontManager.addfont(str(path))\n name = font_manager.FontProperties(fname=str(path)).get_name()\n plt.rcParams[\"font.family\"] = name\n return name\n return plt.rcParams.get(\"font.family\", \"sans-serif\")\n\n\ndef provider_for(model_id: str) -> str | None:\n \"\"\"Map a model identifier to its provider logo (order matters: the\n Codex-harness ``gpt-*`` ids embed the underlying model name).\"\"\"\n m = model_id.lower()\n if \"gemini\" in m:\n return \"gemini\"\n if any(k in m for k in (\"opus\", \"sonnet\", \"claude\", \"anthropic\")):\n return \"anthropic\"\n if any(k in m for k in (\"gpt\", \"openai\", \"codex\")):\n return \"openai\"\n if any(k in m for k in (\"kimi\", \"moonshot\")):\n return \"moonshot\"\n if any(k in m for k in (\"glm\", \"z-ai\", \"zhipu\", \"chatglm\")):\n return \"zai\"\n if \"minimax\" in m:\n return \"minimax\"\n if \"qwen\" in m:\n return \"qwen\"\n if any(k in m for k in (\"devstral\", \"mistral\")):\n return \"mistral\"\n if \"deepseek\" in m:\n return \"deepseek\"\n return None\n\n\ndef load_logo(model_id: str):\n provider = provider_for(model_id)\n if provider is None:\n return None\n path = LOGO_DIR / f\"{provider}.png\"\n if not path.exists():\n return None\n return mpimg.imread(str(path))\n\n\ndef load_task_icon(task_id: str):\n \"\"\"Task icons are named after the task label, e.g. ``deseq.png``.\"\"\"\n path = TASK_ICON_DIR / f\"{task_id}.png\"\n if not path.exists():\n return None\n return mpimg.imread(str(path))\n\n\ndef parse_args():\n parser = argparse.ArgumentParser(description=\"Render task completion heatmap with average bars.\")\n parser.add_argument(\n \"--input-csv\",\n default=\"results/data/all_models_task_completion_rate.csv\",\n help=\"Task-by-model completion-rate CSV.\",\n )\n parser.add_argument(\n \"--output-dir\",\n default=\"results/figures\",\n help=\"Directory for heatmap_bar.png and heatmap_bar.pdf.\",\n )\n parser.add_argument(\n \"--models\",\n nargs=\"*\",\n help=\"Optional model IDs to plot. Defaults to the paper model set when present, otherwise all columns.\",\n )\n parser.add_argument(\"--show\", action=\"store_true\", help=\"Display the plot window after saving.\")\n return parser.parse_args()\n\n\ndef display_label(model: str) -> str:\n return MODEL_LABELS.get(model, model.split(\"/\")[-1] or model)\n\n\nif __name__ == \"__main__\":\n args = parse_args()\n font_name = setup_font(\"medium\")\n\n df = pd.read_csv(args.input_csv)\n df.set_index(\"task_id\", inplace=True)\n\n if args.models:\n best_models = args.models\n else:\n default_models = [model for model in DEFAULT_MODELS if model in df.columns]\n best_models = default_models or df.columns.tolist()\n\n missing_models = [model for model in best_models if model not in df.columns]\n if missing_models:\n print(f\"Skipping missing models: {', '.join(missing_models)}\")\n best_models = [model for model in best_models if model in df.columns]\n if not best_models:\n raise ValueError(\"No requested models were found in the input CSV.\")\n\n df = df[best_models]\n # Rows are models; sort by average completion (descending).\n df = df.T\n averages = df.mean(axis=1)\n sorted_idx = averages.sort_values(ascending=False, kind=\"mergesort\").index\n df = df.loc[sorted_idx]\n averages = averages.loc[sorted_idx]\n\n model_ids = list(df.index)\n task_ids = list(df.columns)\n row_labels = [display_label(model) for model in model_ids]\n\n print(df)\n\n # White -> #13682A gradient; 0% completion is white, 100% is the full color.\n cmap_custom = LinearSegmentedColormap.from_list(\"white_green\", [\"#FFFFFF\", FULL_RATE_COLOR], N=256)\n\n row_count, column_count = df.shape\n axis_fontsize = AXIS_FONTSIZE\n tick_fontsize = 25 if max(row_count, column_count) <= 9 else TICK_FONTSIZE\n annot_fontsize = 22 if max(row_count, column_count) <= 9 else 19\n bar_value_fontsize = 25 if row_count <= 10 else 20\n\n data = df.to_numpy(dtype=float)\n\n # Square cells: 1 data unit per cell on both axes; the figure is sized so\n # the heatmap block is square and the bar panel sits to its right.\n cell_in = 0.62\n heat_w = column_count * cell_in\n heat_h = row_count * cell_in\n bar_w = heat_h * 0.45\n fig_w = heat_w + bar_w + 6.0 # left room for logos + names\n fig_h = heat_h + 3.0 # room for the Task label below\n\n fig = plt.figure(figsize=(fig_w, fig_h))\n # Manual axes placement so the heatmap cells stay perfectly square.\n left_pad = 4.4 / fig_w\n bottom_pad = 1.9 / fig_h\n ax_heat = fig.add_axes([left_pad, bottom_pad, heat_w / fig_w, heat_h / fig_h])\n gap = 0.55 / fig_w\n ax_bar = fig.add_axes([left_pad + heat_w / fig_w + gap, bottom_pad, bar_w / fig_w, heat_h / fig_h])\n\n # --- Heatmap cells ---------------------------------------------------\n ax_heat.imshow(\n data,\n cmap=cmap_custom,\n vmin=0,\n vmax=100,\n aspect=\"equal\",\n extent=(0, column_count, row_count, 0),\n interpolation=\"nearest\",\n )\n ax_heat.set_xlim(0, column_count)\n ax_heat.set_ylim(row_count, 0)\n\n # Black gridlines between cells.\n for x in range(column_count + 1):\n ax_heat.axvline(x, color=\"black\", linewidth=1.0, zorder=3)\n for y in range(row_count + 1):\n ax_heat.axhline(y, color=\"black\", linewidth=1.0, zorder=3)\n\n # Black border around the whole heatmap.\n for spine in ax_heat.spines.values():\n spine.set_visible(True)\n spine.set_color(\"black\")\n spine.set_linewidth(2.2)\n\n # Cell annotations with adaptive contrast (dark cells -> white text).\n full_rgb = np.array(to_rgb(FULL_RATE_COLOR))\n for i in range(row_count):\n for j in range(column_count):\n val = data[i, j]\n if np.isnan(val):\n continue\n # Perceived luminance of the blended cell color.\n cell_rgb = (1 - val / 100) * np.ones(3) + (val / 100) * full_rgb\n lum = 0.299 * cell_rgb[0] + 0.587 * cell_rgb[1] + 0.114 * cell_rgb[2]\n text_color = \"white\" if lum < 0.55 else \"black\"\n ax_heat.text(\n j + 0.5, i + 0.5, f\"{val:.0f}\",\n ha=\"center\", va=\"center\",\n fontsize=annot_fontsize, color=text_color, zorder=4,\n )\n\n # Remove task identifiers on the x-axis (icons will be added later).\n ax_heat.set_xticks([])\n ax_heat.set_yticks([])\n ax_heat.tick_params(length=0)\n\n # Task icons stand in for the x tick labels, one per column. Icons vary in\n # aspect ratio, so each is scaled to fit a common box (never stretched).\n icon_box_in = cell_in * 0.88\n icon_y = row_count + 0.62\n for j, task_id in enumerate(task_ids):\n icon = load_task_icon(task_id)\n if icon is None:\n print(f\"No task icon for '{task_id}'; falling back to text label.\")\n ax_heat.text(\n j + 0.5, icon_y, task_id, rotation=45, ha=\"right\", va=\"center\",\n fontsize=tick_fontsize * 0.7, color=\"black\", clip_on=False,\n )\n continue\n h, w = icon.shape[0], icon.shape[1]\n zoom = min(icon_box_in * 72 / w, icon_box_in * 72 / h)\n ab = AnnotationBbox(\n OffsetImage(icon, zoom=zoom), (j + 0.5, icon_y), frameon=False,\n box_alignment=(0.5, 0.5), annotation_clip=False, zorder=5,\n )\n ax_heat.add_artist(ab)\n\n # \"Task\" label, placed a bit more than one cell height below the heatmap.\n ax_heat.text(\n column_count / 2, row_count + 1.35, \"Task\",\n ha=\"center\", va=\"top\", fontsize=axis_fontsize, color=\"black\",\n clip_on=False,\n )\n # --- Model names + logos (left of the heatmap) -----------------------\n # Order, left to right: \"Model\" label | model name | logo | heatmap.\n # Logos sit in a fixed column hugging the heatmap edge (all logos render\n # at a uniform 256x256, so the column stays aligned).\n from matplotlib.transforms import blended_transform_factory\n blend = blended_transform_factory(ax_heat.transAxes, ax_heat.transData)\n save_dpi = 300\n logo_zoom = (cell_in * save_dpi * 0.74) / 256 / (save_dpi / 72)\n logo_half_w = (cell_in * 0.74 / heat_w) # logo half-width in axes fraction\n logo_center_x = -0.005 - logo_half_w # right edge ~0.005 left of heatmap\n name_right_x = logo_center_x - logo_half_w - 0.007\n name_texts = []\n for i, (mid, label) in enumerate(zip(model_ids, row_labels)):\n y = i + 0.5\n name_texts.append(ax_heat.text(\n name_right_x, y, label, transform=blend,\n ha=\"right\", va=\"center\", fontsize=tick_fontsize, color=\"black\",\n clip_on=False,\n ))\n logo = load_logo(mid)\n if logo is None:\n continue\n im = OffsetImage(logo, zoom=logo_zoom)\n ab = AnnotationBbox(\n im, (logo_center_x, y), xycoords=blend, frameon=False,\n box_alignment=(0.5, 0.5), annotation_clip=False, zorder=5,\n )\n ax_heat.add_artist(ab)\n\n # \"Model\" axis label, just left of the longest model name (measured, so it\n # never collides regardless of font size).\n fig.canvas.draw()\n renderer = fig.canvas.get_renderer()\n axes_inv = ax_heat.transAxes.inverted()\n names_left_x = min(\n axes_inv.transform((t.get_window_extent(renderer=renderer).x0, 0))[0]\n for t in name_texts\n )\n ax_heat.text(\n names_left_x - 0.045, row_count / 2, \"Model\", transform=blend,\n ha=\"center\", va=\"center\", rotation=90,\n fontsize=axis_fontsize, color=\"black\", clip_on=False,\n )\n\n # --- Average bar chart -----------------------------------------------\n bar_positions = np.arange(row_count) + 0.5\n bar_colors = [cmap_custom(val / 100) for val in averages.values]\n bars = ax_bar.barh(\n bar_positions, averages.values, height=0.7,\n color=bar_colors, edgecolor=\"black\", linewidth=1.4,\n )\n\n ax_bar.spines[\"top\"].set_visible(False)\n ax_bar.spines[\"right\"].set_visible(False)\n ax_bar.spines[\"left\"].set_visible(True)\n ax_bar.spines[\"left\"].set_color(\"black\")\n ax_bar.spines[\"bottom\"].set_color(\"black\")\n ax_bar.set_ylim(ax_heat.get_ylim())\n ax_bar.set_yticks([])\n ax_bar.set_xlim(0, 100)\n ax_bar.set_xticks([0, 50, 100])\n ax_bar.tick_params(axis=\"x\", labelsize=tick_fontsize, colors=\"black\")\n ax_bar.text(\n 50, row_count + 1.35, \"Average\",\n ha=\"center\", va=\"top\", fontsize=axis_fontsize, color=\"black\",\n clip_on=False,\n )\n\n for bar, val in zip(bars, averages.values):\n ax_bar.text(\n bar.get_width() + 2, bar.get_y() + bar.get_height() / 2,\n f\"{val:.1f}\", ha=\"left\", va=\"center\",\n fontsize=bar_value_fontsize, color=\"black\",\n )\n\n output_dir = Path(args.output_dir)\n output_dir.mkdir(parents=True, exist_ok=True)\n plt.savefig(output_dir / \"heatmap_bar.png\", dpi=500, bbox_inches=\"tight\", transparent=True)\n plt.savefig(output_dir / \"heatmap_bar.pdf\", dpi=500, bbox_inches=\"tight\", transparent=True)\n if args.show:\n plt.show()\n plt.close(fig)\n"}}}} |