{"repo": "arcadia-1/razavi-bench", "files": {"experiments/2026-06-26-direct-qa/tools/run_deepseek_judge.py": {"34778263afb1481918812e06d19ba2802656696c": {"t": 1785745432.7608633, "n": 1253, "h": "80a6e4eddd43f570", "c": "#!/usr/bin/env python3\nimport argparse\nimport asyncio\n\nfrom judge_common import add_common_args, post_chat, run_judge\n\n\nAPI_URL = \"https://api.deepseek.com/chat/completions\"\n\n\ndef deepseek_chat(\n    api_key: str,\n    model: str,\n    messages: list[dict],\n    timeout: int,\n    max_tokens: int,\n) -> str:\n    payload = {\n        \"model\": model,\n        \"messages\": messages,\n        \"temperature\": 0.0,\n        \"max_tokens\": max_tokens,\n        \"response_format\": {\"type\": \"json_object\"},\n        \"stream\": False,\n    }\n    if model == \"deepseek-v4-pro\":\n        payload[\"thinking\"] = {\"type\": \"disabled\"}\n    body = post_chat(api_key, API_URL, payload, timeout)\n    return body[\"choices\"][0][\"message\"].get(\"content\") or \"\"\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser()\n    add_common_args(\n        parser,\n        default_model=\"deepseek-v4-pro\",\n        default_output_name=\"deepseek-v4-pro\",\n        default_max_tokens=8192,\n    )\n    args = parser.parse_args()\n\n    def chat(api_key: str, model: str, messages: list[dict], timeout: int) -> str:\n        return deepseek_chat(api_key, model, messages, timeout, args.max_tokens)\n\n    asyncio.run(run_judge(args, chat, api_env=\"DEEPSEEK_API_KEY\"))\n\n\nif __name__ == \"__main__\":\n    main()\n"}, "a0ae9a736da2d16ac13b8f309fe0c669f4a1e350": {"t": 1785753159.802748, "n": 1253, "h": "80a6e4eddd43f570", "c": "#!/usr/bin/env python3\nimport argparse\nimport asyncio\n\nfrom judge_common import add_common_args, post_chat, run_judge\n\n\nAPI_URL = \"https://api.deepseek.com/chat/completions\"\n\n\ndef deepseek_chat(\n    api_key: str,\n    model: str,\n    messages: list[dict],\n    timeout: int,\n    max_tokens: int,\n) -> str:\n    payload = {\n        \"model\": model,\n        \"messages\": messages,\n        \"temperature\": 0.0,\n        \"max_tokens\": max_tokens,\n        \"response_format\": {\"type\": \"json_object\"},\n        \"stream\": False,\n    }\n    if model == \"deepseek-v4-pro\":\n        payload[\"thinking\"] = {\"type\": \"disabled\"}\n    body = post_chat(api_key, API_URL, payload, timeout)\n    return body[\"choices\"][0][\"message\"].get(\"content\") or \"\"\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser()\n    add_common_args(\n        parser,\n        default_model=\"deepseek-v4-pro\",\n        default_output_name=\"deepseek-v4-pro\",\n        default_max_tokens=8192,\n    )\n    args = parser.parse_args()\n\n    def chat(api_key: str, model: str, messages: list[dict], timeout: int) -> str:\n        return deepseek_chat(api_key, model, messages, timeout, args.max_tokens)\n\n    asyncio.run(run_judge(args, chat, api_env=\"DEEPSEEK_API_KEY\"))\n\n\nif __name__ == \"__main__\":\n    main()\n"}, "e1187d709727b95a2f8ab9d03b0e74ad51f006d7": {"t": 1786067379.435234, "n": 1253, "h": "80a6e4eddd43f570", "c": "#!/usr/bin/env python3\nimport argparse\nimport asyncio\n\nfrom judge_common import add_common_args, post_chat, run_judge\n\n\nAPI_URL = \"https://api.deepseek.com/chat/completions\"\n\n\ndef deepseek_chat(\n    api_key: str,\n    model: str,\n    messages: list[dict],\n    timeout: int,\n    max_tokens: int,\n) -> str:\n    payload = {\n        \"model\": model,\n        \"messages\": messages,\n        \"temperature\": 0.0,\n        \"max_tokens\": max_tokens,\n        \"response_format\": {\"type\": \"json_object\"},\n        \"stream\": False,\n    }\n    if model == \"deepseek-v4-pro\":\n        payload[\"thinking\"] = {\"type\": \"disabled\"}\n    body = post_chat(api_key, API_URL, payload, timeout)\n    return body[\"choices\"][0][\"message\"].get(\"content\") or \"\"\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser()\n    add_common_args(\n        parser,\n        default_model=\"deepseek-v4-pro\",\n        default_output_name=\"deepseek-v4-pro\",\n        default_max_tokens=8192,\n    )\n    args = parser.parse_args()\n\n    def chat(api_key: str, model: str, messages: list[dict], timeout: int) -> str:\n        return deepseek_chat(api_key, model, messages, timeout, args.max_tokens)\n\n    asyncio.run(run_judge(args, chat, api_env=\"DEEPSEEK_API_KEY\"))\n\n\nif __name__ == \"__main__\":\n    main()\n"}, "04252baa819ef37046352a25538cfbea69400722": {"t": 1786068247.0061364, "n": 1253, "h": "80a6e4eddd43f570", "c": "#!/usr/bin/env python3\nimport argparse\nimport asyncio\n\nfrom judge_common import add_common_args, post_chat, run_judge\n\n\nAPI_URL = \"https://api.deepseek.com/chat/completions\"\n\n\ndef deepseek_chat(\n    api_key: str,\n    model: str,\n    messages: list[dict],\n    timeout: int,\n    max_tokens: int,\n) -> str:\n    payload = {\n        \"model\": model,\n        \"messages\": messages,\n        \"temperature\": 0.0,\n        \"max_tokens\": max_tokens,\n        \"response_format\": {\"type\": \"json_object\"},\n        \"stream\": False,\n    }\n    if model == \"deepseek-v4-pro\":\n        payload[\"thinking\"] = {\"type\": \"disabled\"}\n    body = post_chat(api_key, API_URL, payload, timeout)\n    return body[\"choices\"][0][\"message\"].get(\"content\") or \"\"\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser()\n    add_common_args(\n        parser,\n        default_model=\"deepseek-v4-pro\",\n        default_output_name=\"deepseek-v4-pro\",\n        default_max_tokens=8192,\n    )\n    args = parser.parse_args()\n\n    def chat(api_key: str, model: str, messages: list[dict], timeout: int) -> str:\n        return deepseek_chat(api_key, model, messages, timeout, args.max_tokens)\n\n    asyncio.run(run_judge(args, chat, api_env=\"DEEPSEEK_API_KEY\"))\n\n\nif __name__ == \"__main__\":\n    main()\n"}}, "experiments/2026-08-01-direct-qa/judge_outputs/deepseek-v4-pro-20260801/openrouter-qwen-3.7-plus.metadata.json": {"34778263afb1481918812e06d19ba2802656696c": {"t": 1785746269.7815518, "n": 960, "h": "a91346856f42b62b", "c": "{\n  \"concurrency\": 20,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-08-01-direct-qa/model_outputs/openrouter-qwen-3.7-plus-rollout-*.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"a848a574303143175da2c7835ea303147873e05775ca8671f6c7d106e8414469\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 8192,\n  \"output\": \"experiments/2026-08-01-direct-qa/judge_outputs/deepseek-v4-pro-20260801/openrouter-qwen-3.7-plus.scores.jsonl\",\n  \"record_count\": 150,\n  \"repo\": \".\",\n  \"repo_commit\": \"3e0e57cb210d9ff1ba3e2215b635d9f0bda36a0d\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 150,\n  \"temperature\": 0.0,\n  \"timeout\": 120\n}\n"}, "74e70849e56ebca5d6fd3173029a778f49c794bc": {"t": 1785753162.1882083, "n": 960, "h": "a91346856f42b62b", "c": "{\n  \"concurrency\": 20,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-08-01-direct-qa/model_outputs/openrouter-qwen-3.7-plus-rollout-*.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"a848a574303143175da2c7835ea303147873e05775ca8671f6c7d106e8414469\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 8192,\n  \"output\": \"experiments/2026-08-01-direct-qa/judge_outputs/deepseek-v4-pro-20260801/openrouter-qwen-3.7-plus.scores.jsonl\",\n  \"record_count\": 150,\n  \"repo\": \".\",\n  \"repo_commit\": \"3e0e57cb210d9ff1ba3e2215b635d9f0bda36a0d\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 150,\n  \"temperature\": 0.0,\n  \"timeout\": 120\n}\n"}}, "experiments/2026-08-03-direct-qa/judge_outputs/deepseek-v4-pro-20260803/openrouter-gemini-2-5-flash-lite.metadata.json": {"34778263afb1481918812e06d19ba2802656696c": {"t": 1785746270.1511858, "n": 1030, "h": "16b0a138ddcf8d42", "c": "{\n  \"concurrency\": 20,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-08-03-direct-qa/model_outputs/openrouter-gemini-2-5-flash-lite-rollout-{1,2,3}.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"a848a574303143175da2c7835ea303147873e05775ca8671f6c7d106e8414469\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 8192,\n  \"output\": \"experiments/2026-08-03-direct-qa/judge_outputs/deepseek-v4-pro-20260803/openrouter-gemini-2-5-flash-lite.scores.jsonl\",\n  \"record_count\": 150,\n  \"repo\": \".\",\n  \"repo_commit\": \"e00e8774e9428ef480f591d492a2e3e004628188\",\n  \"repo_dirty\": \" M tools/evaluate_answers.py\\n\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 150,\n  \"temperature\": 0,\n  \"timeout\": 120\n}\n"}, "82f373467aecf84278271fa81a88f29ff1874462": {"t": 1785753161.2681746, "n": 1030, "h": "16b0a138ddcf8d42", "c": "{\n  \"concurrency\": 20,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-08-03-direct-qa/model_outputs/openrouter-gemini-2-5-flash-lite-rollout-{1,2,3}.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"a848a574303143175da2c7835ea303147873e05775ca8671f6c7d106e8414469\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 8192,\n  \"output\": \"experiments/2026-08-03-direct-qa/judge_outputs/deepseek-v4-pro-20260803/openrouter-gemini-2-5-flash-lite.scores.jsonl\",\n  \"record_count\": 150,\n  \"repo\": \".\",\n  \"repo_commit\": \"e00e8774e9428ef480f591d492a2e3e004628188\",\n  \"repo_dirty\": \" M tools/evaluate_answers.py\\n\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 150,\n  \"temperature\": 0,\n  \"timeout\": 120\n}\n"}, "04252baa819ef37046352a25538cfbea69400722": {"t": 1786068407.2864401, "n": 1030, "h": "16b0a138ddcf8d42", "c": "{\n  \"concurrency\": 20,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-08-03-direct-qa/model_outputs/openrouter-gemini-2-5-flash-lite-rollout-{1,2,3}.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"a848a574303143175da2c7835ea303147873e05775ca8671f6c7d106e8414469\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 8192,\n  \"output\": \"experiments/2026-08-03-direct-qa/judge_outputs/deepseek-v4-pro-20260803/openrouter-gemini-2-5-flash-lite.scores.jsonl\",\n  \"record_count\": 150,\n  \"repo\": \".\",\n  \"repo_commit\": \"e00e8774e9428ef480f591d492a2e3e004628188\",\n  \"repo_dirty\": \" M tools/evaluate_answers.py\\n\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 150,\n  \"temperature\": 0,\n  \"timeout\": 120\n}\n"}}, "experiments/2026-07-15-direct-qa/judge_outputs/deepseek-v4-pro-20260716/gpt-5.6-sol-pro-209351-rollout-1.deepseek-v4-pro.scores.jsonl.metadata.json": {"34778263afb1481918812e06d19ba2802656696c": {"t": 1785746272.1342518, "n": 979, "h": "300e7729fe288f89", "c": "{\n  \"concurrency\": 4,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-07-15-direct-qa/model_outputs/gpt-5.6-sol-pro-209351-rollout-1.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"9c6e5bf2435b76b19e13096f4c64f5cd744f68b341810d8e5c296952bf48702a\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 2048,\n  \"output\": \"experiments/2026-07-15-direct-qa/judge_outputs/deepseek-v4-pro-20260716/gpt-5.6-sol-pro-209351-rollout-1.deepseek-v4-pro.scores.jsonl\",\n  \"record_count\": 50,\n  \"repo\": \".\",\n  \"repo_commit\": \"3c3473cff5ee23891a5f6d739564ce1f22b80854\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 50,\n  \"temperature\": 0.0,\n  \"timeout\": 180\n}\n"}, "ca925bbc488bb6ce2637bb9335900d0a8dc19ead": {"t": 1785753161.638622, "n": 979, "h": "300e7729fe288f89", "c": "{\n  \"concurrency\": 4,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-07-15-direct-qa/model_outputs/gpt-5.6-sol-pro-209351-rollout-1.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"9c6e5bf2435b76b19e13096f4c64f5cd744f68b341810d8e5c296952bf48702a\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 2048,\n  \"output\": \"experiments/2026-07-15-direct-qa/judge_outputs/deepseek-v4-pro-20260716/gpt-5.6-sol-pro-209351-rollout-1.deepseek-v4-pro.scores.jsonl\",\n  \"record_count\": 50,\n  \"repo\": \".\",\n  \"repo_commit\": \"3c3473cff5ee23891a5f6d739564ce1f22b80854\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 50,\n  \"temperature\": 0.0,\n  \"timeout\": 180\n}\n"}, "e1187d709727b95a2f8ab9d03b0e74ad51f006d7": {"t": 1786067665.237613, "n": 979, "h": "300e7729fe288f89", "c": "{\n  \"concurrency\": 4,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-07-15-direct-qa/model_outputs/gpt-5.6-sol-pro-209351-rollout-1.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"9c6e5bf2435b76b19e13096f4c64f5cd744f68b341810d8e5c296952bf48702a\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 2048,\n  \"output\": \"experiments/2026-07-15-direct-qa/judge_outputs/deepseek-v4-pro-20260716/gpt-5.6-sol-pro-209351-rollout-1.deepseek-v4-pro.scores.jsonl\",\n  \"record_count\": 50,\n  \"repo\": \".\",\n  \"repo_commit\": \"3c3473cff5ee23891a5f6d739564ce1f22b80854\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 50,\n  \"temperature\": 0.0,\n  \"timeout\": 180\n}\n"}, "04252baa819ef37046352a25538cfbea69400722": {"t": 1786068411.8030844, "n": 979, "h": "300e7729fe288f89", "c": "{\n  \"concurrency\": 4,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-07-15-direct-qa/model_outputs/gpt-5.6-sol-pro-209351-rollout-1.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"9c6e5bf2435b76b19e13096f4c64f5cd744f68b341810d8e5c296952bf48702a\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 2048,\n  \"output\": \"experiments/2026-07-15-direct-qa/judge_outputs/deepseek-v4-pro-20260716/gpt-5.6-sol-pro-209351-rollout-1.deepseek-v4-pro.scores.jsonl\",\n  \"record_count\": 50,\n  \"repo\": \".\",\n  \"repo_commit\": \"3c3473cff5ee23891a5f6d739564ce1f22b80854\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 50,\n  \"temperature\": 0.0,\n  \"timeout\": 180\n}\n"}}, "experiments/2026-08-01-direct-qa/judge_outputs/deepseek-v4-pro-20260801/openrouter-gemini-3-6-flash.metadata.json": {"34778263afb1481918812e06d19ba2802656696c": {"t": 1785746275.9067905, "n": 964, "h": "71f4b27e7ec2cc5e", "c": "{\n  \"concurrency\": 20,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-08-01-direct-qa/model_outputs/openrouter-gemini-3-6-flash-rollout-*.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"a848a574303143175da2c7835ea303147873e05775ca8671f6c7d106e8414469\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 8192,\n  \"output\": \"experiments/2026-08-01-direct-qa/judge_outputs/deepseek-v4-pro-20260801/openrouter-gemini-3-6-flash.scores.jsonl\",\n  \"record_count\": 150,\n  \"repo\": \".\",\n  \"repo_commit\": \"c34c065fba2627067b67fdcf4ba75e786b30c00a\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 150,\n  \"temperature\": 0,\n  \"timeout\": 120\n}\n"}, "5f1a30b64fdf9bb5bce26926fb718aa72c23c5d8": {"t": 1785753160.240289, "n": 964, "h": "71f4b27e7ec2cc5e", "c": "{\n  \"concurrency\": 20,\n  \"failure_count\": 0,\n  \"input\": \"experiments/2026-08-01-direct-qa/model_outputs/openrouter-gemini-3-6-flash-rollout-*.jsonl\",\n  \"json_mode\": true,\n  \"judge_api_format\": \"chat-completions\",\n  \"judge_api_url\": \"https://api.deepseek.com/chat/completions\",\n  \"judge_model\": \"deepseek-v4-pro\",\n  \"judge_script\": \"tools/evaluate_answers.py\",\n  \"judge_script_sha256\": \"a848a574303143175da2c7835ea303147873e05775ca8671f6c7d106e8414469\",\n  \"judge_system_sha256\": \"5dc6bd623ac84598a572a6e06c08516704ef305c07593c1a1f2e3669265d790b\",\n  \"max_retries\": 5,\n  \"max_tokens\": 8192,\n  \"output\": \"experiments/2026-08-01-direct-qa/judge_outputs/deepseek-v4-pro-20260801/openrouter-gemini-3-6-flash.scores.jsonl\",\n  \"record_count\": 150,\n  \"repo\": \".\",\n  \"repo_commit\": \"c34c065fba2627067b67fdcf4ba75e786b30c00a\",\n  \"rubric_sha256\": \"7c95b8e3a1a51fcebb49a1ad05e777ad488f398109bed3af7814ed1709b67a54\",\n  \"scored_count\": 150,\n  \"temperature\": 0,\n  \"timeout\": 120\n}\n"}}}}