1 line
20 KiB
JSON
1 line
20 KiB
JSON
{"repo": "accelerate-data/promptfoo-eval-harness", "files": {"config/sdk-pins.toml": {"f511627c21c3cc2277ef66ea583ad16a0bed09cd": {"t": 1785745370.3244278, "n": 4191, "h": "9ffde083438280f6", "c": "# config/sdk-pins.toml — framework-owned, single source of truth for SDK versions\n# and per-provider env allowlists. See spec §6.4.\n#\n# Consumers CANNOT override per-repo. CVE in a pinned version → harness patch\n# release; consumers run `npm update`.\n\n[openhands_sdk]\nversion = \"1.22.1\"\n# Tools moved out of openhands-sdk into the companion openhands-tools package\n# as of v1.22+. The harness whitelist (tool_registry.py) references the names\n# registered by openhands-tools, so this pin must always accompany the SDK\n# version in the spawn argv. Bump both in lockstep — there is no published\n# compatibility matrix yet, so a smoke run is required on every change.\ntools_version = \"1.23.0\"\npython = \">=3.12,<3.14\"\n# extras: empty — confirmed by spike A.0 (no extras group required).\n# If a future version requires extras (e.g. [\"tools\"]), update here AND\n# every `uv run --with openhands-sdk==...` call site in spec §2.6 / §6.3.\nextras = []\n# Env vars forwarded to the Python subprocess. All other keys matching\n# secret patterns (*_API_KEY, *_TOKEN, *_SECRET, etc.) are stripped. See §7.1.\n# OPENHANDS_BASE_URL and OPENHANDS_MODEL_OVERRIDE: consumer-side .env knobs for\n# pointing the provider at a local/staging gateway and forcing a specific model\n# without editing eval-tiers.toml. See agent_factory.py module docstring.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_BASE_URL\",\n \"OPENHANDS_MODEL_OVERRIDE\",\n \"OPENHANDS_SUPPRESS_BANNER\",\n]\n\n[opencode_cli]\n# OpenCode CLI is system-installed. Harness verifies `opencode --version` against this.\nmin_version = \"0.1.0\"\n# Env vars forwarded to the opencode subprocess (matches opencode-cli-provider.js usage).\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n \"OPENAI_API_KEY\",\n \"OPENCODE_CONFIG\",\n \"XDG_STATE_HOME\",\n]\n\n[claude_agent_sdk]\n# Phase 10 / VD-2174-9 — Claude Agent SDK provider (v1.1.0).\n# Alpha SDK; canary required on each version bump (spec §9.7).\nversion = \"0.2.85\"\npython = \">=3.12,<3.14\"\nextras = []\n# Trimmed env allowlist (Minimalist #6) — Bedrock / Vertex / Foundry backends\n# are out of scope for v1.1.0; re-add when those backends ship.\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n]\n\n[openhands_agent_server]\n# Daemon-mode wrapper provider — see scripts/framework/openhands-agent-server-provider.js.\n# The harness CLI (bin/ad-evals.js) auto-starts the daemon on a dynamic\n# 127.0.0.1 port and injects OPENHANDS_SERVER_URL into the promptfoo\n# subprocess env. The provider reads that env var before falling back\n# to openhands.json.\n# Bumped 2026-05-26 from 1.21.1 → 1.23.1. The 1.21.1 wheel declared an\n# unversioned `openhands-sdk` dependency, so uvx pulled the latest sdk\n# (1.23.x) at runtime → `ImportError: cannot import name 'AgentSettings'\n# from openhands.sdk.settings`. The HTTP/WS surface this provider relies\n# on (POST /api/conversations → `ConversationInfo.id`, WS\n# `/sockets/events/{conversation_id}`, MessageEvent / ActionEvent /\n# ConversationStateUpdateEvent / ConversationErrorEvent payloads) is\n# byte-identical between 1.21.1 and 1.23.1 — this is a pure version\n# refresh, not a provider refactor.\nversion = \"1.23.1\"\n# Locks the openhands-tools companion package version. Source script\n# (templates/scripts/openhands-server.sh) ties the two together; do not\n# split them without verifying upstream compatibility.\ntools_version = \"1.23.1\"\n# Explicit openhands-sdk pin — agent-server-lifecycle.js feeds this into\n# `uvx --with openhands-sdk==<sdk_version>` so the sdk minor never drifts\n# ahead of the agent-server wheel (the 2026-05-26 root cause). Kept here\n# alongside `tools_version` so the lockstep contract test can assert all\n# three move together.\nsdk_version = \"1.23.1\"\npython = \">=3.12,<3.14\"\nextras = []\n# LiteLLM provider keys the daemon child needs to reach model APIs.\n# OPENHANDS_SERVER_URL is set on the promptfoo subprocess by the CLI; it\n# is NOT forwarded into the daemon child env.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENCODE_API_KEY\",\n \"GROQ_API_KEY\",\n \"DEEPSEEK_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_MODEL_OVERRIDE\",\n]\nstartup_timeout_ms = 30000\n"}, "1531b47f4f32e46dae72c2f72c3b73e4691eeb3f": {"t": 1785750811.4828186, "n": 4191, "h": "9ffde083438280f6", "c": "# config/sdk-pins.toml — framework-owned, single source of truth for SDK versions\n# and per-provider env allowlists. See spec §6.4.\n#\n# Consumers CANNOT override per-repo. CVE in a pinned version → harness patch\n# release; consumers run `npm update`.\n\n[openhands_sdk]\nversion = \"1.22.1\"\n# Tools moved out of openhands-sdk into the companion openhands-tools package\n# as of v1.22+. The harness whitelist (tool_registry.py) references the names\n# registered by openhands-tools, so this pin must always accompany the SDK\n# version in the spawn argv. Bump both in lockstep — there is no published\n# compatibility matrix yet, so a smoke run is required on every change.\ntools_version = \"1.23.0\"\npython = \">=3.12,<3.14\"\n# extras: empty — confirmed by spike A.0 (no extras group required).\n# If a future version requires extras (e.g. [\"tools\"]), update here AND\n# every `uv run --with openhands-sdk==...` call site in spec §2.6 / §6.3.\nextras = []\n# Env vars forwarded to the Python subprocess. All other keys matching\n# secret patterns (*_API_KEY, *_TOKEN, *_SECRET, etc.) are stripped. See §7.1.\n# OPENHANDS_BASE_URL and OPENHANDS_MODEL_OVERRIDE: consumer-side .env knobs for\n# pointing the provider at a local/staging gateway and forcing a specific model\n# without editing eval-tiers.toml. See agent_factory.py module docstring.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_BASE_URL\",\n \"OPENHANDS_MODEL_OVERRIDE\",\n \"OPENHANDS_SUPPRESS_BANNER\",\n]\n\n[opencode_cli]\n# OpenCode CLI is system-installed. Harness verifies `opencode --version` against this.\nmin_version = \"0.1.0\"\n# Env vars forwarded to the opencode subprocess (matches opencode-cli-provider.js usage).\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n \"OPENAI_API_KEY\",\n \"OPENCODE_CONFIG\",\n \"XDG_STATE_HOME\",\n]\n\n[claude_agent_sdk]\n# Phase 10 / VD-2174-9 — Claude Agent SDK provider (v1.1.0).\n# Alpha SDK; canary required on each version bump (spec §9.7).\nversion = \"0.2.85\"\npython = \">=3.12,<3.14\"\nextras = []\n# Trimmed env allowlist (Minimalist #6) — Bedrock / Vertex / Foundry backends\n# are out of scope for v1.1.0; re-add when those backends ship.\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n]\n\n[openhands_agent_server]\n# Daemon-mode wrapper provider — see scripts/framework/openhands-agent-server-provider.js.\n# The harness CLI (bin/ad-evals.js) auto-starts the daemon on a dynamic\n# 127.0.0.1 port and injects OPENHANDS_SERVER_URL into the promptfoo\n# subprocess env. The provider reads that env var before falling back\n# to openhands.json.\n# Bumped 2026-05-26 from 1.21.1 → 1.23.1. The 1.21.1 wheel declared an\n# unversioned `openhands-sdk` dependency, so uvx pulled the latest sdk\n# (1.23.x) at runtime → `ImportError: cannot import name 'AgentSettings'\n# from openhands.sdk.settings`. The HTTP/WS surface this provider relies\n# on (POST /api/conversations → `ConversationInfo.id`, WS\n# `/sockets/events/{conversation_id}`, MessageEvent / ActionEvent /\n# ConversationStateUpdateEvent / ConversationErrorEvent payloads) is\n# byte-identical between 1.21.1 and 1.23.1 — this is a pure version\n# refresh, not a provider refactor.\nversion = \"1.23.1\"\n# Locks the openhands-tools companion package version. Source script\n# (templates/scripts/openhands-server.sh) ties the two together; do not\n# split them without verifying upstream compatibility.\ntools_version = \"1.23.1\"\n# Explicit openhands-sdk pin — agent-server-lifecycle.js feeds this into\n# `uvx --with openhands-sdk==<sdk_version>` so the sdk minor never drifts\n# ahead of the agent-server wheel (the 2026-05-26 root cause). Kept here\n# alongside `tools_version` so the lockstep contract test can assert all\n# three move together.\nsdk_version = \"1.23.1\"\npython = \">=3.12,<3.14\"\nextras = []\n# LiteLLM provider keys the daemon child needs to reach model APIs.\n# OPENHANDS_SERVER_URL is set on the promptfoo subprocess by the CLI; it\n# is NOT forwarded into the daemon child env.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENCODE_API_KEY\",\n \"GROQ_API_KEY\",\n \"DEEPSEEK_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_MODEL_OVERRIDE\",\n]\nstartup_timeout_ms = 30000\n"}, "63cf8532bad7b0ab548fd6ff3ec43b886a03bbc5": {"t": 1785750812.5637743, "n": 3265, "h": "4506c18404880451", "c": "# config/sdk-pins.toml — framework-owned, single source of truth for SDK versions\n# and per-provider env allowlists. See spec §6.4.\n#\n# Consumers CANNOT override per-repo. CVE in a pinned version → harness patch\n# release; consumers run `npm update`.\n\n[openhands_sdk]\nversion = \"1.22.1\"\n# Tools moved out of openhands-sdk into the companion openhands-tools package\n# as of v1.22+. The harness whitelist (tool_registry.py) references the names\n# registered by openhands-tools, so this pin must always accompany the SDK\n# version in the spawn argv. Bump both in lockstep — there is no published\n# compatibility matrix yet, so a smoke run is required on every change.\ntools_version = \"1.23.0\"\npython = \">=3.12,<3.14\"\n# extras: empty — confirmed by spike A.0 (no extras group required).\n# If a future version requires extras (e.g. [\"tools\"]), update here AND\n# every `uv run --with openhands-sdk==...` call site in spec §2.6 / §6.3.\nextras = []\n# Env vars forwarded to the Python subprocess. All other keys matching\n# secret patterns (*_API_KEY, *_TOKEN, *_SECRET, etc.) are stripped. See §7.1.\n# OPENHANDS_BASE_URL and OPENHANDS_MODEL_OVERRIDE: consumer-side .env knobs for\n# pointing the provider at a local/staging gateway and forcing a specific model\n# without editing eval-tiers.toml. See agent_factory.py module docstring.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_BASE_URL\",\n \"OPENHANDS_MODEL_OVERRIDE\",\n \"OPENHANDS_SUPPRESS_BANNER\",\n]\n\n[opencode_cli]\n# OpenCode CLI is system-installed. Harness verifies `opencode --version` against this.\nmin_version = \"0.1.0\"\n# Env vars forwarded to the opencode subprocess (matches opencode-cli-provider.js usage).\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n \"OPENAI_API_KEY\",\n \"OPENCODE_CONFIG\",\n \"XDG_STATE_HOME\",\n]\n\n[claude_agent_sdk]\n# Phase 10 / VD-2174-9 — Claude Agent SDK provider (v1.1.0).\n# Alpha SDK; canary required on each version bump (spec §9.7).\nversion = \"0.2.85\"\npython = \">=3.12,<3.14\"\nextras = []\n# Trimmed env allowlist (Minimalist #6) — Bedrock / Vertex / Foundry backends\n# are out of scope for v1.1.0; re-add when those backends ship.\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n]\n\n[openhands_agent_server]\n# Daemon-mode wrapper provider — see scripts/framework/openhands-agent-server-provider.js.\n# The harness CLI (bin/ad-evals.js) auto-starts the daemon on a dynamic\n# 127.0.0.1 port and injects OPENHANDS_SERVER_URL into the promptfoo\n# subprocess env. The provider reads that env var before falling back\n# to openhands.json.\nversion = \"1.21.1\"\n# Locks the openhands-tools companion package version. Source script\n# (templates/scripts/openhands-server.sh) ties the two together; do not\n# split them without verifying upstream compatibility.\ntools_version = \"1.21.1\"\npython = \">=3.12,<3.14\"\nextras = []\n# LiteLLM provider keys the daemon child needs to reach model APIs.\n# OPENHANDS_SERVER_URL is set on the promptfoo subprocess by the CLI; it\n# is NOT forwarded into the daemon child env.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENCODE_API_KEY\",\n \"GROQ_API_KEY\",\n \"DEEPSEEK_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_MODEL_OVERRIDE\",\n]\nstartup_timeout_ms = 30000\n"}, "967bbcd5eaf89f60bfbc8c5d607ebdc2c5340f5e": {"t": 1785750813.6622036, "n": 2224, "h": "13813f36adabe5c2", "c": "# config/sdk-pins.toml — framework-owned, single source of truth for SDK versions\n# and per-provider env allowlists. See spec §6.4.\n#\n# Consumers CANNOT override per-repo. CVE in a pinned version → harness patch\n# release; consumers run `npm update`.\n\n[openhands_sdk]\nversion = \"1.22.1\"\n# Tools moved out of openhands-sdk into the companion openhands-tools package\n# as of v1.22+. The harness whitelist (tool_registry.py) references the names\n# registered by openhands-tools, so this pin must always accompany the SDK\n# version in the spawn argv. Bump both in lockstep — there is no published\n# compatibility matrix yet, so a smoke run is required on every change.\ntools_version = \"1.23.0\"\npython = \">=3.12,<3.14\"\n# extras: empty — confirmed by spike A.0 (no extras group required).\n# If a future version requires extras (e.g. [\"tools\"]), update here AND\n# every `uv run --with openhands-sdk==...` call site in spec §2.6 / §6.3.\nextras = []\n# Env vars forwarded to the Python subprocess. All other keys matching\n# secret patterns (*_API_KEY, *_TOKEN, *_SECRET, etc.) are stripped. See §7.1.\n# OPENHANDS_BASE_URL and OPENHANDS_MODEL_OVERRIDE: consumer-side .env knobs for\n# pointing the provider at a local/staging gateway and forcing a specific model\n# without editing eval-tiers.toml. See agent_factory.py module docstring.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_BASE_URL\",\n \"OPENHANDS_MODEL_OVERRIDE\",\n \"OPENHANDS_SUPPRESS_BANNER\",\n]\n\n[opencode_cli]\n# OpenCode CLI is system-installed. Harness verifies `opencode --version` against this.\nmin_version = \"0.1.0\"\n# Env vars forwarded to the opencode subprocess (matches opencode-cli-provider.js usage).\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n \"OPENAI_API_KEY\",\n \"OPENCODE_CONFIG\",\n \"XDG_STATE_HOME\",\n]\n\n[claude_agent_sdk]\n# Phase 10 / VD-2174-9 — Claude Agent SDK provider (v1.1.0).\n# Alpha SDK; canary required on each version bump (spec §9.7).\nversion = \"0.2.85\"\npython = \">=3.12,<3.14\"\nextras = []\n# Trimmed env allowlist (Minimalist #6) — Bedrock / Vertex / Foundry backends\n# are out of scope for v1.1.0; re-add when those backends ship.\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n]\n"}, "18dacf9d51cb47d4ebf616938674c67070055d77": {"t": 1785750814.7358727, "n": 1936, "h": "e041e980a4e24dc3", "c": "# config/sdk-pins.toml — framework-owned, single source of truth for SDK versions\n# and per-provider env allowlists. See spec §6.4.\n#\n# Consumers CANNOT override per-repo. CVE in a pinned version → harness patch\n# release; consumers run `npm update`.\n\n[openhands_sdk]\nversion = \"1.22.1\"\n# Tools moved out of openhands-sdk into the companion openhands-tools package\n# as of v1.22+. The harness whitelist (tool_registry.py) references the names\n# registered by openhands-tools, so this pin must always accompany the SDK\n# version in the spawn argv. Bump both in lockstep — there is no published\n# compatibility matrix yet, so a smoke run is required on every change.\ntools_version = \"1.23.0\"\npython = \">=3.12,<3.14\"\n# extras: empty — confirmed by spike A.0 (no extras group required).\n# If a future version requires extras (e.g. [\"tools\"]), update here AND\n# every `uv run --with openhands-sdk==...` call site in spec §2.6 / §6.3.\nextras = []\n# Env vars forwarded to the Python subprocess. All other keys matching\n# secret patterns (*_API_KEY, *_TOKEN, *_SECRET, etc.) are stripped. See §7.1.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENHANDS_API_KEY\",\n \"OPENHANDS_SUPPRESS_BANNER\",\n]\n\n[opencode_cli]\n# OpenCode CLI is system-installed. Harness verifies `opencode --version` against this.\nmin_version = \"0.1.0\"\n# Env vars forwarded to the opencode subprocess (matches opencode-cli-provider.js usage).\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n \"OPENAI_API_KEY\",\n \"OPENCODE_CONFIG\",\n \"XDG_STATE_HOME\",\n]\n\n[claude_agent_sdk]\n# Phase 10 / VD-2174-9 — Claude Agent SDK provider (v1.1.0).\n# Alpha SDK; canary required on each version bump (spec §9.7).\nversion = \"0.2.85\"\npython = \">=3.12,<3.14\"\nextras = []\n# Trimmed env allowlist (Minimalist #6) — Bedrock / Vertex / Foundry backends\n# are out of scope for v1.1.0; re-add when those backends ship.\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n]\n"}, "2a3cedd5222d90abf09b5ba3921427d4bfc0d399": {"t": 1785750815.825166, "n": 1510, "h": "7665b7e2f9d9a967", "c": "# config/sdk-pins.toml — framework-owned, single source of truth for SDK versions\n# and per-provider env allowlists. See spec §6.4.\n#\n# Consumers CANNOT override per-repo. CVE in a pinned version → harness patch\n# release; consumers run `npm update`.\n\n[openhands_sdk]\nversion = \"1.22.1\"\npython = \">=3.12,<3.14\"\n# extras: empty — confirmed by spike A.0 (no extras group required).\n# If a future version requires extras (e.g. [\"tools\"]), update here AND\n# every `uv run --with openhands-sdk==...` call site in spec §2.6 / §6.3.\nextras = []\n# Env vars forwarded to the Python subprocess. All other keys matching\n# secret patterns (*_API_KEY, *_TOKEN, *_SECRET, etc.) are stripped. See §7.1.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENHANDS_SUPPRESS_BANNER\",\n]\n\n[opencode_cli]\n# OpenCode CLI is system-installed. Harness verifies `opencode --version` against this.\nmin_version = \"0.1.0\"\n# Env vars forwarded to the opencode subprocess (matches opencode-cli-provider.js usage).\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n \"OPENAI_API_KEY\",\n \"OPENCODE_CONFIG\",\n \"XDG_STATE_HOME\",\n]\n\n[claude_agent_sdk]\n# Phase 10 / VD-2174-9 — Claude Agent SDK provider (v1.1.0).\n# Alpha SDK; canary required on each version bump (spec §9.7).\nversion = \"0.2.85\"\npython = \">=3.12,<3.14\"\nextras = []\n# Trimmed env allowlist (Minimalist #6) — Bedrock / Vertex / Foundry backends\n# are out of scope for v1.1.0; re-add when those backends ship.\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n]\n"}, "846430fcccd1e137014f47861b66718adc04c559": {"t": 1785750816.9161162, "n": 1126, "h": "081ca8674ef8a9c2", "c": "# config/sdk-pins.toml — framework-owned, single source of truth for SDK versions\n# and per-provider env allowlists. See spec §6.4.\n#\n# Consumers CANNOT override per-repo. CVE in a pinned version → harness patch\n# release; consumers run `npm update`.\n\n[openhands_sdk]\nversion = \"1.22.1\"\npython = \">=3.12,<3.14\"\n# extras: empty — confirmed by spike A.0 (no extras group required).\n# If a future version requires extras (e.g. [\"tools\"]), update here AND\n# every `uv run --with openhands-sdk==...` call site in spec §2.6 / §6.3.\nextras = []\n# Env vars forwarded to the Python subprocess. All other keys matching\n# secret patterns (*_API_KEY, *_TOKEN, *_SECRET, etc.) are stripped. See §7.1.\nenv_allowlist = [\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"OPENROUTER_API_KEY\",\n \"OPENHANDS_SUPPRESS_BANNER\",\n]\n\n[opencode_cli]\n# OpenCode CLI is system-installed. Harness verifies `opencode --version` against this.\nmin_version = \"0.1.0\"\n# Env vars forwarded to the opencode subprocess (matches opencode-cli-provider.js usage).\nenv_allowlist = [\n \"ANTHROPIC_API_KEY\",\n \"OPENAI_API_KEY\",\n \"OPENCODE_CONFIG\",\n \"XDG_STATE_HOME\",\n]\n"}}}} |