1 line
57 KiB
JSON
1 line
57 KiB
JSON
{"repo": "bodangren/lending-desk-bench", "files": {"harness/run.sh": {"97fd56674f28e1a0c605ad739fa9c476e9e0fa65": {"t": 1785745960.99471, "n": 27645, "h": "0044c5a085f2041e", "c": "#!/usr/bin/env bash\n# Lending Desk benchmark runner.\n# ./run.sh <provider> <model> <arm: a|b> <run-id>\nset -euo pipefail\nRUN_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9_-]{0,79}$'\n\n\nvalidate_run_id() {\n local run_id=\"$1\"\n [[ \"$run_id\" =~ $RUN_ID_PATTERN && \"$run_id\" != \".\" && \"$run_id\" != \"..\" ]] || {\n echo \"unsafe run id: $run_id\" >&2; exit 2;\n }\n}\n\n\nPROVIDER=\"${1:?usage: run.sh <provider> <model> <a|b> <run-id>}\"\nMODEL=\"${2:?}\"\nARM=\"${3:?}\"\nRUN_ID=\"${4:?}\"\nvalidate_run_id \"$RUN_ID\"\ncase \"$ARM\" in a|b) ;; *) echo \"arm must be a or b\" >&2; exit 2 ;; esac\n\nRUNTIME_PROBE=\"${BENCH_RUNTIME_PROBE:-}\"\ncase \"$RUNTIME_PROBE\" in\n \"\"|ports|container|dependency-copy) ;;\n *) echo \"BENCH_RUNTIME_PROBE must be ports, container, or dependency-copy\" >&2; exit 2 ;;\nesac\n\nRUNTIME_PROBE_HOLD_SECONDS=\"${BENCH_RUNTIME_PROBE_HOLD_SECONDS:-0}\"\nif ! [[ \"$RUNTIME_PROBE_HOLD_SECONDS\" =~ ^[0-9]{1,2}$ ]]; then\n echo \"BENCH_RUNTIME_PROBE_HOLD_SECONDS must be an integer from 0 to 60\" >&2\n exit 2\nfi\nif (( 10#$RUNTIME_PROBE_HOLD_SECONDS > 60 )); then\n echo \"BENCH_RUNTIME_PROBE_HOLD_SECONDS must be an integer from 0 to 60\" >&2\n exit 2\nfi\n\nif [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n if ! [[ \"${BENCH_RUNTIME_PROBE_LABEL:-}\" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$ ]]; then\n echo \"BENCH_RUNTIME_PROBE_LABEL must be a safe Podman label value\" >&2\n exit 2\n fi\nfi\n\nport_for_run() {\n local salt=\"$1\" hash\n hash=$(printf '%s' \"${RUN_ID}:${salt}\" | cksum | awk '{print $1}')\n printf '%s\\n' \"$((20000 + hash % 20000))\"\n}\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")/..\" && pwd)\"\nH=\"$ROOT/harness\"\nWORK=\"$ROOT/runs/$RUN_ID\"\nOUT=\"$WORK/artifacts\"\nCAND=\"$WORK/candidate\"\nTHINKING=\"${THINKING:-max}\" # pi levels: off|minimal|low|medium|high|xhigh|max\nPORT_LOCK_ROOT=\"$ROOT/runs/.port-locks\"\nPORT_LOCKS=()\n\nPORT_LOCK_TOKEN=$(node -e 'process.stdout.write(require(\"node:crypto\").randomBytes(16).toString(\"hex\"))')\n[[ \"$PORT_LOCK_TOKEN\" =~ ^[a-f0-9]{16,}$ ]] || { echo \"could not create port lock token\" >&2; exit 2; }\n\nlock_owner_fields() {\n local lock=\"$1\"\n [ -s \"$lock/owner.json\" ] || return 1\n node -e 'const fs=require(\"node:fs\"); const owner=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); if (owner.schema!==1||!Number.isInteger(owner.pid)||owner.pid<1||typeof owner.token!==\"string\"||!/^[a-f0-9]{16,}$/.test(owner.token)||!Number.isInteger(owner.port)) process.exit(1); console.log(`${owner.pid}\\t${owner.token}\\t${owner.port}`);' \"$lock/owner.json\"\n}\n\nwrite_lock_owner() {\n local lock=\"$1\" port=\"$2\" temporary=\"$lock/owner.json.$PORT_LOCK_TOKEN.tmp\"\n LOCK_PID=\"$$\" LOCK_TOKEN=\"$PORT_LOCK_TOKEN\" LOCK_PORT=\"$port\" node -e 'const owner={schema:1,pid:Number(process.env.LOCK_PID),token:process.env.LOCK_TOKEN,port:Number(process.env.LOCK_PORT)}; if (!Number.isInteger(owner.pid)||!Number.isInteger(owner.port)||!/^[a-f0-9]{16,}$/.test(owner.token)) process.exit(1); process.stdout.write(JSON.stringify(owner));' > \"$temporary\" || return 1\n mv \"$temporary\" \"$lock/owner.json\"\n}\n\nrelease_owned_port_lock() {\n local lock=\"$1\" pid token port\n IFS=$'\\t' read -r pid token port < <(lock_owner_fields \"$lock\" 2>/dev/null) || return 0\n [ \"$pid\" = \"$$\" ] && [ \"$token\" = \"$PORT_LOCK_TOKEN\" ] || return 0\n rm -f \"$lock/owner.json\"\n rmdir \"$lock\" 2>/dev/null || true\n}\n\nstale_port_lock() {\n local lock=\"$1\" port=\"$2\" pid token owner_port\n IFS=$'\\t' read -r pid token owner_port < <(lock_owner_fields \"$lock\" 2>/dev/null) || return 1\n [ \"$owner_port\" = \"$port\" ] || return 1\n kill -0 \"$pid\" 2>/dev/null && return 1\n return 0\n}\n\nreclaim_stale_port_lock() {\n local lock=\"$1\" port=\"$2\" retired\n stale_port_lock \"$lock\" \"$port\" || return 1\n retired=\"$lock.stale.$PORT_LOCK_TOKEN\"\n mv \"$lock\" \"$retired\" 2>/dev/null || return 1\n rm -rf \"$retired\"\n return 0\n}\nrelease_port_locks() {\n local lock\n for lock in \"${PORT_LOCKS[@]}\"; do\n release_owned_port_lock \"$lock\"\n done\n}\ntrap release_port_locks EXIT INT TERM\n\nvalid_port_number() {\n local port=\"$1\"\n [[ \"$port\" =~ ^[0-9]{1,5}$ ]] && [ \"$port\" -ge 1024 ] && [ \"$port\" -le 65535 ]\n}\n\nreclaim_stale_port_locks() {\n local lock port\n for lock in \"$PORT_LOCK_ROOT\"/*; do\n [ -d \"$lock\" ] || continue\n port=\"${lock##*/}\"\n valid_port_number \"$port\" || continue\n reclaim_stale_port_lock \"$lock\" \"$port\" || true\n done\n}\n\nport_is_available() {\n local port=\"$1\"\n ! ss -ltnH \"sport = :$port\" 2>/dev/null | grep -q .\n}\n\nclaim_port() {\n local salt=\"$1\" requested=\"$2\" attempt port lock\n command -v ss >/dev/null || { echo \"ss is required for safe port allocation\" >&2; return 1; }\n mkdir -p \"$PORT_LOCK_ROOT\"\n reclaim_stale_port_locks\n if [ -n \"$requested\" ]; then\n valid_port_number \"$requested\" || { echo \"invalid requested port: $requested\" >&2; return 1; }\n fi\n for ((attempt = 0; attempt < 20000; attempt++)); do\n if [ -n \"$requested\" ]; then\n port=\"$requested\"\n else\n port=$((20000 + ($(port_for_run \"$salt:$attempt\") - 20000 + attempt) % 20000))\n fi\n lock=\"$PORT_LOCK_ROOT/$port\"\n if mkdir \"$lock\" 2>/dev/null; then\n if ! write_lock_owner \"$lock\" \"$port\"; then\n rmdir \"$lock\" 2>/dev/null || true\n continue\n fi\n if port_is_available \"$port\"; then\n PORT_LOCKS+=(\"$lock\")\n CLAIMED_PORT=\"$port\"\n return 0\n fi\n release_owned_port_lock \"$lock\"\n else\n for settle in 1 2 3; do\n [ -s \"$lock/owner.json\" ] && break\n sleep 0.01\n done\n if reclaim_stale_port_lock \"$lock\" \"$port\"; then\n continue\n fi\n fi\n [ -z \"$requested\" ] || break\n done\n echo \"unable to reserve a free $salt port\" >&2\n return 1\n}\n\nREQUESTED_API_PORT=\"${API_PORT:-}\"\nREQUESTED_E2E_PORT=\"${E2E_PORT:-}\"\nif [ \"$RUNTIME_PROBE\" != \"container\" ] && [ \"$RUNTIME_PROBE\" != \"dependency-copy\" ]; then\n claim_port api \"$REQUESTED_API_PORT\"\n API_PORT=\"$CLAIMED_PORT\"\n claim_port e2e \"$REQUESTED_E2E_PORT\"\n E2E_PORT=\"$CLAIMED_PORT\"\nfi\n\nruntime_port_probe() {\n mkdir -p \"$OUT\"\n printf '{\"schema\":1,\"api_port\":%s,\"e2e_port\":%s}\\n' \"$API_PORT\" \"$E2E_PORT\" \\\n > \"$OUT/runtime-port-probe.json\"\n if (( 10#$RUNTIME_PROBE_HOLD_SECONDS > 0 )); then\n sleep \"$RUNTIME_PROBE_HOLD_SECONDS\"\n fi\n exit 0\n}\n\nassert_candidate_contract() {\n if ! \"$H/node_modules/.bin/tsx\" \"$H/candidate-contract.ts\" \"$ROOT/fixture\" \"$CAND\" \\\n > \"$OUT/candidate-contract.json\"; then\n echo \"candidate filesystem contract rejected the run:\" >&2\n node -e 'try { const c = JSON.parse(require(\"node:fs\").readFileSync(process.argv[1], \"utf8\")); console.error((c.reasons || []).join(\"; \") || \"no contract reason was emitted\"); } catch { console.error(\"the candidate contract did not emit valid JSON\"); }' \\\n \"$OUT/candidate-contract.json\" >&2 || true\n return 1\n fi\n}\n\nwrite_provenance() {\n local candidate_hash fixture_hash runner_hash suite_hash hashes contract_fields identity_hash\n [ -s \"$OUT/candidate-contract.json\" ] || return 1\n [ -s \"$OUT/execution-identity.json\" ] || return 1\n [ -s \"$OUT/executor-cohort.json\" ] || return 1\n contract_fields=$(node -e 'const c = JSON.parse(require(\"node:fs\").readFileSync(process.argv[1], \"utf8\")); const hex = /^[a-f0-9]{64}$/; if (c.schema !== 1 || c.valid !== true || !hex.test(c.candidate_sha256) || !hex.test(c.fixture_protected_sha256)) process.exit(1); process.stdout.write(`${c.candidate_sha256}\\t${c.fixture_protected_sha256}`);' \"$OUT/candidate-contract.json\") || return 1\n IFS=$'\\t' read -r candidate_hash fixture_hash <<< \"$contract_fields\"\n hashes=$(\"$H/node_modules/.bin/tsx\" \"$H/provenance.ts\" --shell) || return 1\n IFS=$'\\t' read -r runner_hash suite_hash <<< \"$hashes\"\n identity_hash=$(\"$H/node_modules/.bin/tsx\" \"$H/execution-identity.ts\" --fingerprint \"$OUT/execution-identity.json\") || return 1\n [ -n \"$runner_hash\" ] && [ -n \"$suite_hash\" ] && [[ \"$identity_hash\" =~ ^[a-f0-9]{64}$ ]] || return 1\n ARM_VALUE=\"$ARM\" RUN_ID_VALUE=\"$RUN_ID\" MODE_VALUE=\"${RUN_MODE:-unknown}\" MODEL_VALUE=\"$PROVIDER/$MODEL\" CANDIDATE_HASH=\"$candidate_hash\" FIXTURE_HASH=\"$fixture_hash\" RUNNER_HASH=\"$runner_hash\" SUITE_HASH=\"$suite_hash\" ISOLATED_VALUE=\"${AGENT_ISOLATED:-false}\" IDENTITY_PATH=\"$OUT/execution-identity.json\" COHORT_PATH=\"$OUT/executor-cohort.json\" IDENTITY_HASH=\"$identity_hash\" node -e 'const fs=require(\"node:fs\"); const e=process.env; if (!/^(a|b)$/.test(e.ARM_VALUE) || !/^(agent|overlay)$/.test(e.MODE_VALUE)) throw new Error(\"invalid provenance identity\"); if ((e.MODE_VALUE === \"agent\") !== (e.ISOLATED_VALUE === \"true\")) throw new Error(\"invalid provenance isolation\"); const executor=JSON.parse(fs.readFileSync(e.IDENTITY_PATH,\"utf8\")),cohort=JSON.parse(fs.readFileSync(e.COHORT_PATH,\"utf8\")); if (cohort.schema!==1||typeof cohort.id!==\"string\"||!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(cohort.id)) throw new Error(\"invalid executor cohort\"); console.log(JSON.stringify({ schema: 4, run_id: e.RUN_ID_VALUE, arm: e.ARM_VALUE, model: e.MODEL_VALUE, mode: e.MODE_VALUE, candidate_sha256: e.CANDIDATE_HASH, fixture_protected_sha256: e.FIXTURE_HASH, runner_sha256: e.RUNNER_HASH, suite_sha256: e.SUITE_HASH, agent_isolated: e.ISOLATED_VALUE === \"true\", candidate_contract_schema: 1, cohort_id: cohort.id, executor_cohort: cohort, executor, execution_identity_sha256: e.IDENTITY_HASH }));' > \"$OUT/provenance.json\"\n}\n\njson_array() {\n node -e 'process.stdout.write(JSON.stringify(process.argv.slice(1)))' \"$@\"\n}\n\n\nwrite_executor_cohort() {\n [ -s \"$OUT/execution-identity.json\" ] || return 1\n EXECUTION_IDENTITY_PATH=\"$OUT/execution-identity.json\" EXECUTOR_COHORT_ID=\"${BENCH_EXECUTOR_COHORT_ID:-}\" node -e 'const fs=require(\"node:fs\"),crypto=require(\"node:crypto\"),e=process.env,identity=JSON.parse(fs.readFileSync(e.EXECUTION_IDENTITY_PATH,\"utf8\")); const strings=v=>Array.isArray(v)&&v.length>0&&v.every(x=>typeof x===\"string\"&&x.length>0); const stable=v=>v===null||[\"boolean\",\"number\",\"string\"].includes(typeof v)?JSON.stringify(v):Array.isArray(v)?\"[\"+v.map(stable).join(\",\")+\"]\":typeof v===\"object\"?\"{\"+Object.keys(v).sort().map(k=>JSON.stringify(k)+\":\"+stable(v[k])).join(\",\")+\"}\":JSON.stringify(null); if (identity.schema!==1||!/^(a|b)$/.test(identity.arm)||typeof identity.run_id!==\"string\") throw new Error(\"execution identity is malformed\"); let cohort; if (identity.kind===\"not-invoked\") cohort={schema:1,kind:\"not-invoked\",arms:{}}; else { if (!strings(identity.runtime_flags)||!strings(identity.cli_flags)||!identity.skills||typeof identity.skill_tree_sha256!==\"string\") throw new Error(\"execution identity flags or skills are malformed\"); cohort={schema:1,image_reference:identity.image_reference,image_identity:identity.image_identity,pi:identity.pi,runtime_flags:identity.runtime_flags,cli_flags:identity.cli_flags,arms:{[identity.arm]:{skills:identity.skills,skill_tree_sha256:identity.skill_tree_sha256}}}; } const derived=\"executor-cohort-\"+crypto.createHash(\"sha256\").update(stable(cohort)).digest(\"hex\").slice(0,24); const id=e.EXECUTOR_COHORT_ID||derived; if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(id)) throw new Error(\"executor cohort ID is invalid\"); cohort.id=id; console.log(JSON.stringify(cohort));' > \"$OUT/executor-cohort.json\"\n}\nwrite_overlay_identity() {\n EXECUTION_RUN_ID=\"$RUN_ID\" EXECUTION_ARM=\"$ARM\" EXECUTION_MODE=overlay \\\n \"$H/node_modules/.bin/tsx\" \"$H/execution-identity.ts\" --from-env > \"$OUT/execution-identity.json\"\n write_executor_cohort\n}\n\nwrite_agent_identity() {\n local runtime_flags cli_flags\n runtime_flags=$(json_array \"${podman_args[@]}\" --entrypoint /usr/bin/timeout) || return 1\n cli_flags=$(json_array \"${AGENT_TIMEOUT:-3600}\" /opt/pi/bin/pi \\\n --provider \"$PROVIDER\" --model \"$MODEL\" --print --mode json --no-session --thinking \"$THINKING\" --approve \\\n --tools read,write,edit,bash,grep,find,ls \"${pi_flags[@]}\" \"$PROMPT\") || return 1\n EXECUTION_RUN_ID=\"$RUN_ID\" EXECUTION_ARM=\"$ARM\" EXECUTION_MODE=agent \\\n EXECUTION_IMAGE_REFERENCE=\"$pi_image\" EXECUTION_PI_PATH=\"$pi_root/bin/pi\" EXECUTION_SKILL_ROOT=\"$skill_root\" \\\n EXECUTION_RUNTIME_FLAGS=\"$runtime_flags\" EXECUTION_CLI_FLAGS=\"$cli_flags\" \\\n \"$H/node_modules/.bin/tsx\" \"$H/execution-identity.ts\" --from-env > \"$OUT/execution-identity.json\"\n write_executor_cohort\n}\napply_overlay() {\n local overlay=\"$1\" rel\n [ -d \"$overlay\" ] || { echo \"overlay does not exist: $overlay\" >&2; return 1; }\n for rel in \"${WRITABLE_CANDIDATE_PATHS[@]}\"; do\n [ -f \"$overlay/$rel\" ] || continue\n mkdir -p \"$(dirname \"$CAND/$rel\")\"\n cp \"$overlay/$rel\" \"$CAND/$rel\"\n done\n}\n\nwrite_invalid() {\n local reason=\"$1\"\n RUN_ID=\"$RUN_ID\" MODELFULL=\"$PROVIDER/$MODEL\" ARM=\"$ARM\" INVALID_REASON=\"$reason\" \\\n node -e 'console.log(JSON.stringify({schema:2,valid:false,trusted:false,publishable:false,rankable:false,provenance_valid:false,suite_current:false,blocked_by_gate:false,invalid_reasons:[process.env.INVALID_REASON],total:null,run_id:process.env.RUN_ID,model:process.env.MODELFULL,arm:process.env.ARM,invalid:process.env.INVALID_REASON}, null, 2))' \\\n > \"$OUT/score.json\"\n}\n\nhydrate_candidate_dependencies() {\n # Pi gets fixture dependencies read-only. The host grader receives a private\n # physical copy: the lockfile cache is intentionally not an online requirement,\n # and hard links would let one candidate mutate the fixture or another run.\n rm -rf \"$CAND/node_modules\"\n cp -a \"$ROOT/fixture/node_modules\" \"$CAND/node_modules\"\n}\n\n\nrun_container_probe() {\n local probe_image=\"${BENCH_AGENT_IMAGE:-docker.io/library/node@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3}\"\n local skill_root=\"${BENCH_SKILL_ROOT:-/home/daniel-bo/.agents/skills}\"\n local probe_pi_root=\"${BENCH_PI_ROOT:-/home/daniel-bo/.local/share/fnm/node-versions/v22.22.3/installation}\"\n local probe_loopback_port=\"${BENCH_RUNTIME_PROBE_LOOPBACK_PORT:-9}\"\n local probe_host_home_path=\"${BENCH_RUNTIME_PROBE_HOST_HOME_PATH:-${HOME:-/home/daniel-bo}}\"\n local probe_sibling_run_path=\"${BENCH_RUNTIME_PROBE_SIBLING_RUN_PATH:-$ROOT/runs}\"\n local skill arm_a_agents_backup=\"\" probe_contract=phase9 probe_rc\n local -a podman_args\n\n command -v podman >/dev/null || { echo \"podman is required for isolated runtime probes\" >&2; return 2; }\n [ -d \"$ROOT/fixture/node_modules\" ] || { echo \"fixture node_modules is unavailable\" >&2; return 2; }\n [ -f \"$H/runtime-container-probe.cjs\" ] || { echo \"runtime probe helper is unavailable\" >&2; return 2; }\n [ -d \"$probe_pi_root\" ] || { echo \"runtime probe Pi root is unavailable: $probe_pi_root\" >&2; return 2; }\n if ! [[ \"$probe_loopback_port\" =~ ^[0-9]{1,5}$ ]] || (( 10#$probe_loopback_port < 1 || 10#$probe_loopback_port > 65535 )); then\n echo \"BENCH_RUNTIME_PROBE_LOOPBACK_PORT must be a valid TCP port\" >&2\n return 2\n fi\n for probe_path in \"$probe_host_home_path\" \"$probe_sibling_run_path\"; do\n if [[ \"$probe_path\" != /* ]] || [ ! -d \"$probe_path\" ]; then\n echo \"runtime probe host paths must be existing absolute directories\" >&2\n return 2\n fi\n done\n [ \"$MODEL\" = \"phase6-runtime-probe\" ] && probe_contract=phase6\n mkdir -p \"$CAND/node_modules\"\n podman_args=(run --rm --pull=never --read-only\n --tmpfs /tmp:rw,nosuid,nodev,noexec,size=512m\n --cap-drop=ALL\n --security-opt=no-new-privileges\n --userns=keep-id\n --user \"$(id -u):$(id -g)\"\n --network slirp4netns:allow_host_loopback=false\n --workdir /workspace\n --env HOME=/tmp\n --env XDG_CONFIG_HOME=/tmp/config\n --env XDG_CACHE_HOME=/tmp/cache\n --env \"BENCH_RUNTIME_PROBE_CONTRACT=$probe_contract\"\n --env \"BENCH_RUNTIME_PROBE_LOOPBACK_PORT=$probe_loopback_port\"\n --env \"BENCH_RUNTIME_PROBE_HOST_HOME_PATH=$probe_host_home_path\"\n --env \"BENCH_RUNTIME_PROBE_SIBLING_RUN_PATH=$probe_sibling_run_path\"\n --mount \"type=bind,src=$CAND,dst=/workspace,rw\"\n --mount \"type=bind,src=$ROOT/fixture/node_modules,dst=/workspace/node_modules,ro\"\n --mount \"type=bind,src=$H/runtime-container-probe.cjs,dst=/opt/runtime-container-probe.cjs,ro\"\n --mount \"type=bind,src=$probe_pi_root,dst=/opt/pi,ro\")\n\n if [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n podman_args+=(--label \"lending-desk.runtime-probe=$BENCH_RUNTIME_PROBE_LABEL\")\n fi\n\n case \"$ARM\" in\n a)\n arm_a_agents_backup=\"$OUT/arm-a-AGENTS.md\"\n ;;\n b)\n for skill in measure next-best-practices vercel-react-best-practices vercel-composition-patterns build-graph; do\n [ -f \"$skill_root/$skill/SKILL.md\" ] || { echo \"required Arm B skill unavailable: $skill\" >&2; return 2; }\n podman_args+=(--mount \"type=bind,src=$skill_root/$skill,dst=/opt/skills/$skill,ro\")\n done\n ;;\n *) echo \"arm must be a or b\" >&2; return 2 ;;\n esac\n\n if [ -n \"$arm_a_agents_backup\" ]; then\n [ -f \"$CAND/AGENTS.md\" ] || { echo \"Arm A candidate AGENTS.md is missing\" >&2; return 2; }\n mv \"$CAND/AGENTS.md\" \"$arm_a_agents_backup\"\n fi\n\n if podman \"${podman_args[@]}\" \"$probe_image\" node /opt/runtime-container-probe.cjs; then\n probe_rc=0\n else\n probe_rc=$?\n fi\n if [ -n \"$arm_a_agents_backup\" ]; then\n mv \"$arm_a_agents_backup\" \"$CAND/AGENTS.md\" || return 2\n fi\n return \"$probe_rc\"\n}\n\nrun_agent_container() {\n local pi_root=\"${BENCH_PI_ROOT:-/home/daniel-bo/.local/share/fnm/node-versions/v22.22.3/installation}\"\n # BENCH_AGENT_IMAGE remains an explicit override; this default is the locally\n # verified Node image digest, so --pull=never cannot silently change it.\n local pi_image=\"${BENCH_AGENT_IMAGE:-docker.io/library/node@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3}\"\n local skill_root=\"${BENCH_SKILL_ROOT:-/home/daniel-bo/.agents/skills}\"\n local secret_name=\"${BENCH_PROVIDER_ENV:-}\" skill arm_a_agents_backup=\"\" agent_rc\n local -a podman_args pi_flags\n\n command -v podman >/dev/null || { echo \"podman is required for isolated agent runs\" >&2; return 2; }\n [ -x \"$pi_root/bin/pi\" ] || { echo \"Pi installation unavailable: $pi_root\" >&2; return 2; }\n [ -d \"$ROOT/fixture/node_modules\" ] || { echo \"fixture node_modules is unavailable\" >&2; return 2; }\n\n # A model gets exactly one provider credential, never the process's full secret set.\n # Custom providers must nominate one of these names explicitly via BENCH_PROVIDER_ENV.\n case \"$PROVIDER\" in\n openai) secret_name=\"${BENCH_PROVIDER_ENV:-OPENAI_API_KEY}\" ;;\n anthropic) secret_name=\"${BENCH_PROVIDER_ENV:-ANTHROPIC_API_KEY}\" ;;\n openrouter) secret_name=\"${BENCH_PROVIDER_ENV:-OPENROUTER_API_KEY}\" ;;\n google|gemini) secret_name=\"${BENCH_PROVIDER_ENV:-GOOGLE_API_KEY}\" ;;\n deepseek) secret_name=\"${BENCH_PROVIDER_ENV:-DEEPSEEK_API_KEY}\" ;;\n groq) secret_name=\"${BENCH_PROVIDER_ENV:-GROQ_API_KEY}\" ;;\n mistral) secret_name=\"${BENCH_PROVIDER_ENV:-MISTRAL_API_KEY}\" ;;\n xai) secret_name=\"${BENCH_PROVIDER_ENV:-XAI_API_KEY}\" ;;\n together) secret_name=\"${BENCH_PROVIDER_ENV:-TOGETHER_API_KEY}\" ;;\n cerebras) secret_name=\"${BENCH_PROVIDER_ENV:-CEREBRAS_API_KEY}\" ;;\n fireworks) secret_name=\"${BENCH_PROVIDER_ENV:-FIREWORKS_API_KEY}\" ;;\n xiaomi) secret_name=\"${BENCH_PROVIDER_ENV:-XIAOMI_API_KEY}\" ;;\n esac\n case \"$secret_name\" in\n ANTHROPIC_API_KEY|OPENAI_API_KEY|OPENROUTER_API_KEY|GOOGLE_API_KEY|GEMINI_API_KEY|DEEPSEEK_API_KEY|GROQ_API_KEY|MISTRAL_API_KEY|XAI_API_KEY|TOGETHER_API_KEY|CEREBRAS_API_KEY|FIREWORKS_API_KEY|XIAOMI_API_KEY) ;;\n *) echo \"set BENCH_PROVIDER_ENV to one allowlisted provider key for $PROVIDER\" >&2; return 2 ;;\n esac\n mkdir -p \"$CAND/node_modules\"\n podman_args=(run --rm --pull=never --read-only\n --tmpfs /tmp:rw,nosuid,nodev,noexec,size=512m\n --cap-drop=ALL\n --security-opt=no-new-privileges\n --userns=keep-id\n --user \"$(id -u):$(id -g)\"\n --network slirp4netns:allow_host_loopback=false\n --workdir /workspace\n --env HOME=/tmp\n --env XDG_CONFIG_HOME=/tmp/config\n --env XDG_CACHE_HOME=/tmp/cache\n --env PI_CODING_AGENT_DIR=/tmp/pi\n --env PI_TELEMETRY=0\n --env \"$secret_name\"\n --mount \"type=bind,src=$CAND,dst=/workspace,rw\"\n --mount \"type=bind,src=$ROOT/fixture/node_modules,dst=/workspace/node_modules,ro\"\n --mount \"type=bind,src=$pi_root,dst=/opt/pi,ro\")\n\n if [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n podman_args+=(--label \"lending-desk.runtime-probe=$BENCH_RUNTIME_PROBE_LABEL\")\n fi\n\n case \"$ARM\" in\n a)\n pi_flags=(--no-skills --no-extensions --no-context-files --no-prompt-templates --no-themes)\n arm_a_agents_backup=\"$OUT/arm-a-AGENTS.md\"\n ;;\n b)\n # Arm B reads the candidate's fixture AGENTS.md and the explicit skills below.\n pi_flags=(--no-extensions --no-prompt-templates --no-themes)\n for skill in measure next-best-practices vercel-react-best-practices vercel-composition-patterns build-graph; do\n [ -f \"$skill_root/$skill/SKILL.md\" ] || { echo \"required Arm B skill unavailable: $skill\" >&2; return 2; }\n podman_args+=(--mount \"type=bind,src=$skill_root/$skill,dst=/opt/skills/$skill,ro\")\n pi_flags+=(--skill \"/opt/skills/$skill\")\n done\n ;;\n *) echo \"arm must be a or b\" >&2; return 2 ;;\n esac\n\n write_agent_identity || { echo \"could not resolve agent execution identity\" >&2; return 2; }\n [ -n \"${!secret_name:-}\" ] || { echo \"missing required provider environment: $secret_name\" >&2; return 2; }\n if [ -n \"$arm_a_agents_backup\" ]; then\n [ -f \"$CAND/AGENTS.md\" ] || { echo \"Arm A candidate AGENTS.md is missing\" >&2; return 2; }\n mv \"$CAND/AGENTS.md\" \"$arm_a_agents_backup\"\n fi\n\n if podman \"${podman_args[@]}\" --entrypoint /usr/bin/timeout \"$pi_image\" \\\n \"${AGENT_TIMEOUT:-3600}\" /opt/pi/bin/pi \\\n --provider \"$PROVIDER\" --model \"$MODEL\" \\\n --print --mode json --no-session --thinking \"$THINKING\" --approve \\\n --tools read,write,edit,bash,grep,find,ls \\\n \"${pi_flags[@]}\" \"$PROMPT\"; then\n agent_rc=0\n else\n agent_rc=$?\n fi\n if [ -n \"$arm_a_agents_backup\" ]; then\n mv \"$arm_a_agents_backup\" \"$CAND/AGENTS.md\" || return 2\n fi\n return \"$agent_rc\"\n}\n\n\n# 0. Host readiness. Checked before the agent runs so a loaded machine costs\n# nothing rather than an hour of tokens followed by void timing results.\n# 6 means \"busy but BENCH_IGNORE_LOAD=1\" — proceed; grade.sh flags the record.\nif \"$H/preflight.sh\" \"$RUN_ID\"; then\n PF=0\nelse\n PF=$?\nfi\n[ \"$PF\" -eq 0 ] || [ \"$PF\" -eq 6 ] || exit 5\n\nrm -rf \"$WORK\"; mkdir -p \"$OUT\"\nif [ \"$RUNTIME_PROBE\" = \"ports\" ]; then\n runtime_port_probe\nfi\n\n# The bounded port probe must publish its allocation promptly. Normal runs\n# still resolve the candidate contract before any fixture or agent work.\nWRITABLE_PATHS=$(\"$H/node_modules/.bin/tsx\" \"$H/candidate-contract.ts\" --writable-paths) || { echo \"candidate contract allowlist is unavailable\" >&2; exit 2; }\nmapfile -t WRITABLE_CANDIDATE_PATHS <<< \"$WRITABLE_PATHS\"\n[ \"${#WRITABLE_CANDIDATE_PATHS[@]}\" -gt 0 ] || { echo \"candidate contract allowlist is empty\" >&2; exit 2; }\nunset WRITABLE_PATHS\n\necho \"[$RUN_ID] $PROVIDER/$MODEL arm=$ARM\"\n\n# 1. Fresh fixture, byte-identical between arms.\nmkdir -p \"$CAND\"\ntar -C \"$ROOT/fixture\" --exclude=node_modules --exclude=.next --exclude=.git \\\n --exclude='*.tsbuildinfo' -cf - . | tar -C \"$CAND\" -xf -\n# Dependencies are mounted read-only for the agent and symlinked only after verification.\n\n\nPROMPT='Read measure/tracks/lending_desk/spec.md and measure/tracks/lending_desk/plan.md, then implement the plan. Mark each plan task complete as you finish it. Run \"npm run typecheck\" and \"npm run build\" and make sure both pass before you stop.'\n\nif [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n echo \"[$RUN_ID] running isolated container runtime probe\"\n if run_container_probe > \"$OUT/session.json\" 2> \"$OUT/session.err\"; then\n exit 0\n else\n PROBE_RC=$?\n exit \"$PROBE_RC\"\n fi\nfi\n\nif [ \"$RUNTIME_PROBE\" = \"dependency-copy\" ]; then\n echo \"[$RUN_ID] running bounded dependency-copy probe\"\n if hydrate_candidate_dependencies; then\n exit 0\n else\n echo \"[$RUN_ID] dependency-copy probe could not create a private copy\" >&2\n exit 2\n fi\nfi\n# 2. Agent.\necho \"[$RUN_ID] invoking agent...\"\nSTART=$(date +%s)\nif [ \"${AGENT_SKIP:-0}\" = \"1\" ]; then\n RUN_MODE=\"overlay\"; AGENT_ISOLATED=false\n if ! apply_overlay \"${OVERLAY:-$ROOT/reference}\"; then\n write_invalid \"overlay could not be applied\"\n exit 3\n fi\n if ! write_overlay_identity; then\n write_invalid \"overlay execution identity could not be resolved\"\n exit 3\n fi\n AGENT_EXIT=0\n echo \"[$RUN_ID] AGENT_SKIP=1 - overlay ${OVERLAY:-$ROOT/reference} applied\"\nelse\n RUN_MODE=\"agent\"; AGENT_ISOLATED=true\n if run_agent_container > \"$OUT/session.json\" 2> \"$OUT/session.err\"; then\n AGENT_EXIT=0\n else\n AGENT_EXIT=$?\n fi\nfi\nEND=$(date +%s)\necho \"[$RUN_ID] agent exit=$AGENT_EXIT in $((END-START))s\"\n\nif ! assert_candidate_contract; then\n echo \"[$RUN_ID] ABORT: candidate violates the filesystem contract\" >&2\n write_invalid \"candidate violates the filesystem contract\"\n exit 3\nfi\nif ! write_provenance; then\n echo \"[$RUN_ID] ABORT: candidate provenance could not be written\" >&2\n write_invalid \"candidate provenance could not be written\"\n exit 3\nfi\n\ndiff -ru \"$ROOT/fixture\" \"$CAND\" -x node_modules -x .next -x '*.tsbuildinfo' \\\n > \"$OUT/candidate.diff\" 2>/dev/null || true\n\n# 2b. Did the agent actually run? `pi` exits 0 on a provider error (no API key,\n# lapsed token, refused request, unknown model id), which used to produce a graded\n# score for an untouched fixture.\n#\n# Two signals, both about work actually produced:\n# - empty diff (a-gpt-5.6-luna, 2026-07-31: lapsed OAuth token)\n# - no output tokens (a-ling-3.0-flash-r1, 2026-08-01: an unresolvable model id\n# returns one empty turn. Its diff was 1 line, so `! -s` did\n# not fire, the gate went green, and it scored 3.1 for $0.)\n#\n# Do NOT add a check on pi's \"not found for provider ... using custom model id\"\n# warning. It is emitted for any id outside pi's local registry and says nothing\n# about whether the call succeeds: on 2026-08-01 both inclusionai/ling-3.0-flash\n# (0 tokens, dead) and inclusionai/ling-3.0-flash:free (219MB transcript, 23KB diff,\n# a real submission) printed it verbatim. Grepping for it aborted the working run.\nif [ \"${AGENT_SKIP:-0}\" != \"1\" ]; then\n INVALID=\"\"\n [ -s \"$OUT/candidate.diff\" ] || INVALID=\"agent produced no changes\"\n OUT_TOKENS=$(\"$H/node_modules/.bin/tsx\" \"$H/usage.ts\" \"$OUT/session.json\" 2>/dev/null \\\n | python3 -c \"import json,sys; print(json.load(sys.stdin).get('output',0))\" 2>/dev/null || echo 0)\n [ \"${OUT_TOKENS:-0}\" -gt 0 ] 2>/dev/null || INVALID=\"agent emitted no output tokens\"\n\n if [ -n \"$INVALID\" ]; then\n echo \"[$RUN_ID] ABORT: $INVALID\" >&2\n { grep -o '\"errorMessage\":\"[^\"]*\"' \"$OUT/session.json\" 2>/dev/null | sort -u | head -3 >&2; } || true\n tail -3 \"$OUT/session.err\" >&2 2>/dev/null || true\n write_invalid \"$INVALID\"\n exit 3\n fi\nfi\n\nif ! hydrate_candidate_dependencies; then\n echo \"[$RUN_ID] ABORT: unable to create private candidate dependencies\" >&2\n write_invalid \"unable to create private candidate dependencies\"\n exit 3\nfi\n\n# here is what let it drift out of sync with collect.ts.\nif AGENT_EXIT=\"$AGENT_EXIT\" THINKING=\"$THINKING\" API_PORT=\"$API_PORT\" E2E_PORT=\"$E2E_PORT\" \\\n \"$H/grade.sh\" \"$RUN_ID\" \"$PROVIDER/$MODEL\" \"$ARM\" \"$((END-START))\"; then\n GRADE_RC=0\nelse\n GRADE_RC=$?\nfi\n\n# 6. Compress the transcript. pi repeats the whole message on every message_update,\n# so these reach ~1GB each (a-deepseek-v4-flash: 939MB) and the matrix runs every\n# model ten times. Done after scoring, since extractUsage reads it; usage.ts accepts\n# the .gz form so a later standalone regrade still reports usage.\nif [ -s \"$OUT/session.json\" ]; then\n gzip -f \"$OUT/session.json\" && echo \"[$RUN_ID] transcript compressed -> session.json.gz\"\nfi\nexit $GRADE_RC\n"}, "00e8ffe12740b44d91ade040fe2255ff60b531eb": {"t": 1785758313.6269252, "n": 27645, "h": "0044c5a085f2041e", "c": "#!/usr/bin/env bash\n# Lending Desk benchmark runner.\n# ./run.sh <provider> <model> <arm: a|b> <run-id>\nset -euo pipefail\nRUN_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9_-]{0,79}$'\n\n\nvalidate_run_id() {\n local run_id=\"$1\"\n [[ \"$run_id\" =~ $RUN_ID_PATTERN && \"$run_id\" != \".\" && \"$run_id\" != \"..\" ]] || {\n echo \"unsafe run id: $run_id\" >&2; exit 2;\n }\n}\n\n\nPROVIDER=\"${1:?usage: run.sh <provider> <model> <a|b> <run-id>}\"\nMODEL=\"${2:?}\"\nARM=\"${3:?}\"\nRUN_ID=\"${4:?}\"\nvalidate_run_id \"$RUN_ID\"\ncase \"$ARM\" in a|b) ;; *) echo \"arm must be a or b\" >&2; exit 2 ;; esac\n\nRUNTIME_PROBE=\"${BENCH_RUNTIME_PROBE:-}\"\ncase \"$RUNTIME_PROBE\" in\n \"\"|ports|container|dependency-copy) ;;\n *) echo \"BENCH_RUNTIME_PROBE must be ports, container, or dependency-copy\" >&2; exit 2 ;;\nesac\n\nRUNTIME_PROBE_HOLD_SECONDS=\"${BENCH_RUNTIME_PROBE_HOLD_SECONDS:-0}\"\nif ! [[ \"$RUNTIME_PROBE_HOLD_SECONDS\" =~ ^[0-9]{1,2}$ ]]; then\n echo \"BENCH_RUNTIME_PROBE_HOLD_SECONDS must be an integer from 0 to 60\" >&2\n exit 2\nfi\nif (( 10#$RUNTIME_PROBE_HOLD_SECONDS > 60 )); then\n echo \"BENCH_RUNTIME_PROBE_HOLD_SECONDS must be an integer from 0 to 60\" >&2\n exit 2\nfi\n\nif [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n if ! [[ \"${BENCH_RUNTIME_PROBE_LABEL:-}\" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$ ]]; then\n echo \"BENCH_RUNTIME_PROBE_LABEL must be a safe Podman label value\" >&2\n exit 2\n fi\nfi\n\nport_for_run() {\n local salt=\"$1\" hash\n hash=$(printf '%s' \"${RUN_ID}:${salt}\" | cksum | awk '{print $1}')\n printf '%s\\n' \"$((20000 + hash % 20000))\"\n}\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")/..\" && pwd)\"\nH=\"$ROOT/harness\"\nWORK=\"$ROOT/runs/$RUN_ID\"\nOUT=\"$WORK/artifacts\"\nCAND=\"$WORK/candidate\"\nTHINKING=\"${THINKING:-max}\" # pi levels: off|minimal|low|medium|high|xhigh|max\nPORT_LOCK_ROOT=\"$ROOT/runs/.port-locks\"\nPORT_LOCKS=()\n\nPORT_LOCK_TOKEN=$(node -e 'process.stdout.write(require(\"node:crypto\").randomBytes(16).toString(\"hex\"))')\n[[ \"$PORT_LOCK_TOKEN\" =~ ^[a-f0-9]{16,}$ ]] || { echo \"could not create port lock token\" >&2; exit 2; }\n\nlock_owner_fields() {\n local lock=\"$1\"\n [ -s \"$lock/owner.json\" ] || return 1\n node -e 'const fs=require(\"node:fs\"); const owner=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); if (owner.schema!==1||!Number.isInteger(owner.pid)||owner.pid<1||typeof owner.token!==\"string\"||!/^[a-f0-9]{16,}$/.test(owner.token)||!Number.isInteger(owner.port)) process.exit(1); console.log(`${owner.pid}\\t${owner.token}\\t${owner.port}`);' \"$lock/owner.json\"\n}\n\nwrite_lock_owner() {\n local lock=\"$1\" port=\"$2\" temporary=\"$lock/owner.json.$PORT_LOCK_TOKEN.tmp\"\n LOCK_PID=\"$$\" LOCK_TOKEN=\"$PORT_LOCK_TOKEN\" LOCK_PORT=\"$port\" node -e 'const owner={schema:1,pid:Number(process.env.LOCK_PID),token:process.env.LOCK_TOKEN,port:Number(process.env.LOCK_PORT)}; if (!Number.isInteger(owner.pid)||!Number.isInteger(owner.port)||!/^[a-f0-9]{16,}$/.test(owner.token)) process.exit(1); process.stdout.write(JSON.stringify(owner));' > \"$temporary\" || return 1\n mv \"$temporary\" \"$lock/owner.json\"\n}\n\nrelease_owned_port_lock() {\n local lock=\"$1\" pid token port\n IFS=$'\\t' read -r pid token port < <(lock_owner_fields \"$lock\" 2>/dev/null) || return 0\n [ \"$pid\" = \"$$\" ] && [ \"$token\" = \"$PORT_LOCK_TOKEN\" ] || return 0\n rm -f \"$lock/owner.json\"\n rmdir \"$lock\" 2>/dev/null || true\n}\n\nstale_port_lock() {\n local lock=\"$1\" port=\"$2\" pid token owner_port\n IFS=$'\\t' read -r pid token owner_port < <(lock_owner_fields \"$lock\" 2>/dev/null) || return 1\n [ \"$owner_port\" = \"$port\" ] || return 1\n kill -0 \"$pid\" 2>/dev/null && return 1\n return 0\n}\n\nreclaim_stale_port_lock() {\n local lock=\"$1\" port=\"$2\" retired\n stale_port_lock \"$lock\" \"$port\" || return 1\n retired=\"$lock.stale.$PORT_LOCK_TOKEN\"\n mv \"$lock\" \"$retired\" 2>/dev/null || return 1\n rm -rf \"$retired\"\n return 0\n}\nrelease_port_locks() {\n local lock\n for lock in \"${PORT_LOCKS[@]}\"; do\n release_owned_port_lock \"$lock\"\n done\n}\ntrap release_port_locks EXIT INT TERM\n\nvalid_port_number() {\n local port=\"$1\"\n [[ \"$port\" =~ ^[0-9]{1,5}$ ]] && [ \"$port\" -ge 1024 ] && [ \"$port\" -le 65535 ]\n}\n\nreclaim_stale_port_locks() {\n local lock port\n for lock in \"$PORT_LOCK_ROOT\"/*; do\n [ -d \"$lock\" ] || continue\n port=\"${lock##*/}\"\n valid_port_number \"$port\" || continue\n reclaim_stale_port_lock \"$lock\" \"$port\" || true\n done\n}\n\nport_is_available() {\n local port=\"$1\"\n ! ss -ltnH \"sport = :$port\" 2>/dev/null | grep -q .\n}\n\nclaim_port() {\n local salt=\"$1\" requested=\"$2\" attempt port lock\n command -v ss >/dev/null || { echo \"ss is required for safe port allocation\" >&2; return 1; }\n mkdir -p \"$PORT_LOCK_ROOT\"\n reclaim_stale_port_locks\n if [ -n \"$requested\" ]; then\n valid_port_number \"$requested\" || { echo \"invalid requested port: $requested\" >&2; return 1; }\n fi\n for ((attempt = 0; attempt < 20000; attempt++)); do\n if [ -n \"$requested\" ]; then\n port=\"$requested\"\n else\n port=$((20000 + ($(port_for_run \"$salt:$attempt\") - 20000 + attempt) % 20000))\n fi\n lock=\"$PORT_LOCK_ROOT/$port\"\n if mkdir \"$lock\" 2>/dev/null; then\n if ! write_lock_owner \"$lock\" \"$port\"; then\n rmdir \"$lock\" 2>/dev/null || true\n continue\n fi\n if port_is_available \"$port\"; then\n PORT_LOCKS+=(\"$lock\")\n CLAIMED_PORT=\"$port\"\n return 0\n fi\n release_owned_port_lock \"$lock\"\n else\n for settle in 1 2 3; do\n [ -s \"$lock/owner.json\" ] && break\n sleep 0.01\n done\n if reclaim_stale_port_lock \"$lock\" \"$port\"; then\n continue\n fi\n fi\n [ -z \"$requested\" ] || break\n done\n echo \"unable to reserve a free $salt port\" >&2\n return 1\n}\n\nREQUESTED_API_PORT=\"${API_PORT:-}\"\nREQUESTED_E2E_PORT=\"${E2E_PORT:-}\"\nif [ \"$RUNTIME_PROBE\" != \"container\" ] && [ \"$RUNTIME_PROBE\" != \"dependency-copy\" ]; then\n claim_port api \"$REQUESTED_API_PORT\"\n API_PORT=\"$CLAIMED_PORT\"\n claim_port e2e \"$REQUESTED_E2E_PORT\"\n E2E_PORT=\"$CLAIMED_PORT\"\nfi\n\nruntime_port_probe() {\n mkdir -p \"$OUT\"\n printf '{\"schema\":1,\"api_port\":%s,\"e2e_port\":%s}\\n' \"$API_PORT\" \"$E2E_PORT\" \\\n > \"$OUT/runtime-port-probe.json\"\n if (( 10#$RUNTIME_PROBE_HOLD_SECONDS > 0 )); then\n sleep \"$RUNTIME_PROBE_HOLD_SECONDS\"\n fi\n exit 0\n}\n\nassert_candidate_contract() {\n if ! \"$H/node_modules/.bin/tsx\" \"$H/candidate-contract.ts\" \"$ROOT/fixture\" \"$CAND\" \\\n > \"$OUT/candidate-contract.json\"; then\n echo \"candidate filesystem contract rejected the run:\" >&2\n node -e 'try { const c = JSON.parse(require(\"node:fs\").readFileSync(process.argv[1], \"utf8\")); console.error((c.reasons || []).join(\"; \") || \"no contract reason was emitted\"); } catch { console.error(\"the candidate contract did not emit valid JSON\"); }' \\\n \"$OUT/candidate-contract.json\" >&2 || true\n return 1\n fi\n}\n\nwrite_provenance() {\n local candidate_hash fixture_hash runner_hash suite_hash hashes contract_fields identity_hash\n [ -s \"$OUT/candidate-contract.json\" ] || return 1\n [ -s \"$OUT/execution-identity.json\" ] || return 1\n [ -s \"$OUT/executor-cohort.json\" ] || return 1\n contract_fields=$(node -e 'const c = JSON.parse(require(\"node:fs\").readFileSync(process.argv[1], \"utf8\")); const hex = /^[a-f0-9]{64}$/; if (c.schema !== 1 || c.valid !== true || !hex.test(c.candidate_sha256) || !hex.test(c.fixture_protected_sha256)) process.exit(1); process.stdout.write(`${c.candidate_sha256}\\t${c.fixture_protected_sha256}`);' \"$OUT/candidate-contract.json\") || return 1\n IFS=$'\\t' read -r candidate_hash fixture_hash <<< \"$contract_fields\"\n hashes=$(\"$H/node_modules/.bin/tsx\" \"$H/provenance.ts\" --shell) || return 1\n IFS=$'\\t' read -r runner_hash suite_hash <<< \"$hashes\"\n identity_hash=$(\"$H/node_modules/.bin/tsx\" \"$H/execution-identity.ts\" --fingerprint \"$OUT/execution-identity.json\") || return 1\n [ -n \"$runner_hash\" ] && [ -n \"$suite_hash\" ] && [[ \"$identity_hash\" =~ ^[a-f0-9]{64}$ ]] || return 1\n ARM_VALUE=\"$ARM\" RUN_ID_VALUE=\"$RUN_ID\" MODE_VALUE=\"${RUN_MODE:-unknown}\" MODEL_VALUE=\"$PROVIDER/$MODEL\" CANDIDATE_HASH=\"$candidate_hash\" FIXTURE_HASH=\"$fixture_hash\" RUNNER_HASH=\"$runner_hash\" SUITE_HASH=\"$suite_hash\" ISOLATED_VALUE=\"${AGENT_ISOLATED:-false}\" IDENTITY_PATH=\"$OUT/execution-identity.json\" COHORT_PATH=\"$OUT/executor-cohort.json\" IDENTITY_HASH=\"$identity_hash\" node -e 'const fs=require(\"node:fs\"); const e=process.env; if (!/^(a|b)$/.test(e.ARM_VALUE) || !/^(agent|overlay)$/.test(e.MODE_VALUE)) throw new Error(\"invalid provenance identity\"); if ((e.MODE_VALUE === \"agent\") !== (e.ISOLATED_VALUE === \"true\")) throw new Error(\"invalid provenance isolation\"); const executor=JSON.parse(fs.readFileSync(e.IDENTITY_PATH,\"utf8\")),cohort=JSON.parse(fs.readFileSync(e.COHORT_PATH,\"utf8\")); if (cohort.schema!==1||typeof cohort.id!==\"string\"||!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(cohort.id)) throw new Error(\"invalid executor cohort\"); console.log(JSON.stringify({ schema: 4, run_id: e.RUN_ID_VALUE, arm: e.ARM_VALUE, model: e.MODEL_VALUE, mode: e.MODE_VALUE, candidate_sha256: e.CANDIDATE_HASH, fixture_protected_sha256: e.FIXTURE_HASH, runner_sha256: e.RUNNER_HASH, suite_sha256: e.SUITE_HASH, agent_isolated: e.ISOLATED_VALUE === \"true\", candidate_contract_schema: 1, cohort_id: cohort.id, executor_cohort: cohort, executor, execution_identity_sha256: e.IDENTITY_HASH }));' > \"$OUT/provenance.json\"\n}\n\njson_array() {\n node -e 'process.stdout.write(JSON.stringify(process.argv.slice(1)))' \"$@\"\n}\n\n\nwrite_executor_cohort() {\n [ -s \"$OUT/execution-identity.json\" ] || return 1\n EXECUTION_IDENTITY_PATH=\"$OUT/execution-identity.json\" EXECUTOR_COHORT_ID=\"${BENCH_EXECUTOR_COHORT_ID:-}\" node -e 'const fs=require(\"node:fs\"),crypto=require(\"node:crypto\"),e=process.env,identity=JSON.parse(fs.readFileSync(e.EXECUTION_IDENTITY_PATH,\"utf8\")); const strings=v=>Array.isArray(v)&&v.length>0&&v.every(x=>typeof x===\"string\"&&x.length>0); const stable=v=>v===null||[\"boolean\",\"number\",\"string\"].includes(typeof v)?JSON.stringify(v):Array.isArray(v)?\"[\"+v.map(stable).join(\",\")+\"]\":typeof v===\"object\"?\"{\"+Object.keys(v).sort().map(k=>JSON.stringify(k)+\":\"+stable(v[k])).join(\",\")+\"}\":JSON.stringify(null); if (identity.schema!==1||!/^(a|b)$/.test(identity.arm)||typeof identity.run_id!==\"string\") throw new Error(\"execution identity is malformed\"); let cohort; if (identity.kind===\"not-invoked\") cohort={schema:1,kind:\"not-invoked\",arms:{}}; else { if (!strings(identity.runtime_flags)||!strings(identity.cli_flags)||!identity.skills||typeof identity.skill_tree_sha256!==\"string\") throw new Error(\"execution identity flags or skills are malformed\"); cohort={schema:1,image_reference:identity.image_reference,image_identity:identity.image_identity,pi:identity.pi,runtime_flags:identity.runtime_flags,cli_flags:identity.cli_flags,arms:{[identity.arm]:{skills:identity.skills,skill_tree_sha256:identity.skill_tree_sha256}}}; } const derived=\"executor-cohort-\"+crypto.createHash(\"sha256\").update(stable(cohort)).digest(\"hex\").slice(0,24); const id=e.EXECUTOR_COHORT_ID||derived; if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(id)) throw new Error(\"executor cohort ID is invalid\"); cohort.id=id; console.log(JSON.stringify(cohort));' > \"$OUT/executor-cohort.json\"\n}\nwrite_overlay_identity() {\n EXECUTION_RUN_ID=\"$RUN_ID\" EXECUTION_ARM=\"$ARM\" EXECUTION_MODE=overlay \\\n \"$H/node_modules/.bin/tsx\" \"$H/execution-identity.ts\" --from-env > \"$OUT/execution-identity.json\"\n write_executor_cohort\n}\n\nwrite_agent_identity() {\n local runtime_flags cli_flags\n runtime_flags=$(json_array \"${podman_args[@]}\" --entrypoint /usr/bin/timeout) || return 1\n cli_flags=$(json_array \"${AGENT_TIMEOUT:-3600}\" /opt/pi/bin/pi \\\n --provider \"$PROVIDER\" --model \"$MODEL\" --print --mode json --no-session --thinking \"$THINKING\" --approve \\\n --tools read,write,edit,bash,grep,find,ls \"${pi_flags[@]}\" \"$PROMPT\") || return 1\n EXECUTION_RUN_ID=\"$RUN_ID\" EXECUTION_ARM=\"$ARM\" EXECUTION_MODE=agent \\\n EXECUTION_IMAGE_REFERENCE=\"$pi_image\" EXECUTION_PI_PATH=\"$pi_root/bin/pi\" EXECUTION_SKILL_ROOT=\"$skill_root\" \\\n EXECUTION_RUNTIME_FLAGS=\"$runtime_flags\" EXECUTION_CLI_FLAGS=\"$cli_flags\" \\\n \"$H/node_modules/.bin/tsx\" \"$H/execution-identity.ts\" --from-env > \"$OUT/execution-identity.json\"\n write_executor_cohort\n}\napply_overlay() {\n local overlay=\"$1\" rel\n [ -d \"$overlay\" ] || { echo \"overlay does not exist: $overlay\" >&2; return 1; }\n for rel in \"${WRITABLE_CANDIDATE_PATHS[@]}\"; do\n [ -f \"$overlay/$rel\" ] || continue\n mkdir -p \"$(dirname \"$CAND/$rel\")\"\n cp \"$overlay/$rel\" \"$CAND/$rel\"\n done\n}\n\nwrite_invalid() {\n local reason=\"$1\"\n RUN_ID=\"$RUN_ID\" MODELFULL=\"$PROVIDER/$MODEL\" ARM=\"$ARM\" INVALID_REASON=\"$reason\" \\\n node -e 'console.log(JSON.stringify({schema:2,valid:false,trusted:false,publishable:false,rankable:false,provenance_valid:false,suite_current:false,blocked_by_gate:false,invalid_reasons:[process.env.INVALID_REASON],total:null,run_id:process.env.RUN_ID,model:process.env.MODELFULL,arm:process.env.ARM,invalid:process.env.INVALID_REASON}, null, 2))' \\\n > \"$OUT/score.json\"\n}\n\nhydrate_candidate_dependencies() {\n # Pi gets fixture dependencies read-only. The host grader receives a private\n # physical copy: the lockfile cache is intentionally not an online requirement,\n # and hard links would let one candidate mutate the fixture or another run.\n rm -rf \"$CAND/node_modules\"\n cp -a \"$ROOT/fixture/node_modules\" \"$CAND/node_modules\"\n}\n\n\nrun_container_probe() {\n local probe_image=\"${BENCH_AGENT_IMAGE:-docker.io/library/node@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3}\"\n local skill_root=\"${BENCH_SKILL_ROOT:-/home/daniel-bo/.agents/skills}\"\n local probe_pi_root=\"${BENCH_PI_ROOT:-/home/daniel-bo/.local/share/fnm/node-versions/v22.22.3/installation}\"\n local probe_loopback_port=\"${BENCH_RUNTIME_PROBE_LOOPBACK_PORT:-9}\"\n local probe_host_home_path=\"${BENCH_RUNTIME_PROBE_HOST_HOME_PATH:-${HOME:-/home/daniel-bo}}\"\n local probe_sibling_run_path=\"${BENCH_RUNTIME_PROBE_SIBLING_RUN_PATH:-$ROOT/runs}\"\n local skill arm_a_agents_backup=\"\" probe_contract=phase9 probe_rc\n local -a podman_args\n\n command -v podman >/dev/null || { echo \"podman is required for isolated runtime probes\" >&2; return 2; }\n [ -d \"$ROOT/fixture/node_modules\" ] || { echo \"fixture node_modules is unavailable\" >&2; return 2; }\n [ -f \"$H/runtime-container-probe.cjs\" ] || { echo \"runtime probe helper is unavailable\" >&2; return 2; }\n [ -d \"$probe_pi_root\" ] || { echo \"runtime probe Pi root is unavailable: $probe_pi_root\" >&2; return 2; }\n if ! [[ \"$probe_loopback_port\" =~ ^[0-9]{1,5}$ ]] || (( 10#$probe_loopback_port < 1 || 10#$probe_loopback_port > 65535 )); then\n echo \"BENCH_RUNTIME_PROBE_LOOPBACK_PORT must be a valid TCP port\" >&2\n return 2\n fi\n for probe_path in \"$probe_host_home_path\" \"$probe_sibling_run_path\"; do\n if [[ \"$probe_path\" != /* ]] || [ ! -d \"$probe_path\" ]; then\n echo \"runtime probe host paths must be existing absolute directories\" >&2\n return 2\n fi\n done\n [ \"$MODEL\" = \"phase6-runtime-probe\" ] && probe_contract=phase6\n mkdir -p \"$CAND/node_modules\"\n podman_args=(run --rm --pull=never --read-only\n --tmpfs /tmp:rw,nosuid,nodev,noexec,size=512m\n --cap-drop=ALL\n --security-opt=no-new-privileges\n --userns=keep-id\n --user \"$(id -u):$(id -g)\"\n --network slirp4netns:allow_host_loopback=false\n --workdir /workspace\n --env HOME=/tmp\n --env XDG_CONFIG_HOME=/tmp/config\n --env XDG_CACHE_HOME=/tmp/cache\n --env \"BENCH_RUNTIME_PROBE_CONTRACT=$probe_contract\"\n --env \"BENCH_RUNTIME_PROBE_LOOPBACK_PORT=$probe_loopback_port\"\n --env \"BENCH_RUNTIME_PROBE_HOST_HOME_PATH=$probe_host_home_path\"\n --env \"BENCH_RUNTIME_PROBE_SIBLING_RUN_PATH=$probe_sibling_run_path\"\n --mount \"type=bind,src=$CAND,dst=/workspace,rw\"\n --mount \"type=bind,src=$ROOT/fixture/node_modules,dst=/workspace/node_modules,ro\"\n --mount \"type=bind,src=$H/runtime-container-probe.cjs,dst=/opt/runtime-container-probe.cjs,ro\"\n --mount \"type=bind,src=$probe_pi_root,dst=/opt/pi,ro\")\n\n if [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n podman_args+=(--label \"lending-desk.runtime-probe=$BENCH_RUNTIME_PROBE_LABEL\")\n fi\n\n case \"$ARM\" in\n a)\n arm_a_agents_backup=\"$OUT/arm-a-AGENTS.md\"\n ;;\n b)\n for skill in measure next-best-practices vercel-react-best-practices vercel-composition-patterns build-graph; do\n [ -f \"$skill_root/$skill/SKILL.md\" ] || { echo \"required Arm B skill unavailable: $skill\" >&2; return 2; }\n podman_args+=(--mount \"type=bind,src=$skill_root/$skill,dst=/opt/skills/$skill,ro\")\n done\n ;;\n *) echo \"arm must be a or b\" >&2; return 2 ;;\n esac\n\n if [ -n \"$arm_a_agents_backup\" ]; then\n [ -f \"$CAND/AGENTS.md\" ] || { echo \"Arm A candidate AGENTS.md is missing\" >&2; return 2; }\n mv \"$CAND/AGENTS.md\" \"$arm_a_agents_backup\"\n fi\n\n if podman \"${podman_args[@]}\" \"$probe_image\" node /opt/runtime-container-probe.cjs; then\n probe_rc=0\n else\n probe_rc=$?\n fi\n if [ -n \"$arm_a_agents_backup\" ]; then\n mv \"$arm_a_agents_backup\" \"$CAND/AGENTS.md\" || return 2\n fi\n return \"$probe_rc\"\n}\n\nrun_agent_container() {\n local pi_root=\"${BENCH_PI_ROOT:-/home/daniel-bo/.local/share/fnm/node-versions/v22.22.3/installation}\"\n # BENCH_AGENT_IMAGE remains an explicit override; this default is the locally\n # verified Node image digest, so --pull=never cannot silently change it.\n local pi_image=\"${BENCH_AGENT_IMAGE:-docker.io/library/node@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3}\"\n local skill_root=\"${BENCH_SKILL_ROOT:-/home/daniel-bo/.agents/skills}\"\n local secret_name=\"${BENCH_PROVIDER_ENV:-}\" skill arm_a_agents_backup=\"\" agent_rc\n local -a podman_args pi_flags\n\n command -v podman >/dev/null || { echo \"podman is required for isolated agent runs\" >&2; return 2; }\n [ -x \"$pi_root/bin/pi\" ] || { echo \"Pi installation unavailable: $pi_root\" >&2; return 2; }\n [ -d \"$ROOT/fixture/node_modules\" ] || { echo \"fixture node_modules is unavailable\" >&2; return 2; }\n\n # A model gets exactly one provider credential, never the process's full secret set.\n # Custom providers must nominate one of these names explicitly via BENCH_PROVIDER_ENV.\n case \"$PROVIDER\" in\n openai) secret_name=\"${BENCH_PROVIDER_ENV:-OPENAI_API_KEY}\" ;;\n anthropic) secret_name=\"${BENCH_PROVIDER_ENV:-ANTHROPIC_API_KEY}\" ;;\n openrouter) secret_name=\"${BENCH_PROVIDER_ENV:-OPENROUTER_API_KEY}\" ;;\n google|gemini) secret_name=\"${BENCH_PROVIDER_ENV:-GOOGLE_API_KEY}\" ;;\n deepseek) secret_name=\"${BENCH_PROVIDER_ENV:-DEEPSEEK_API_KEY}\" ;;\n groq) secret_name=\"${BENCH_PROVIDER_ENV:-GROQ_API_KEY}\" ;;\n mistral) secret_name=\"${BENCH_PROVIDER_ENV:-MISTRAL_API_KEY}\" ;;\n xai) secret_name=\"${BENCH_PROVIDER_ENV:-XAI_API_KEY}\" ;;\n together) secret_name=\"${BENCH_PROVIDER_ENV:-TOGETHER_API_KEY}\" ;;\n cerebras) secret_name=\"${BENCH_PROVIDER_ENV:-CEREBRAS_API_KEY}\" ;;\n fireworks) secret_name=\"${BENCH_PROVIDER_ENV:-FIREWORKS_API_KEY}\" ;;\n xiaomi) secret_name=\"${BENCH_PROVIDER_ENV:-XIAOMI_API_KEY}\" ;;\n esac\n case \"$secret_name\" in\n ANTHROPIC_API_KEY|OPENAI_API_KEY|OPENROUTER_API_KEY|GOOGLE_API_KEY|GEMINI_API_KEY|DEEPSEEK_API_KEY|GROQ_API_KEY|MISTRAL_API_KEY|XAI_API_KEY|TOGETHER_API_KEY|CEREBRAS_API_KEY|FIREWORKS_API_KEY|XIAOMI_API_KEY) ;;\n *) echo \"set BENCH_PROVIDER_ENV to one allowlisted provider key for $PROVIDER\" >&2; return 2 ;;\n esac\n mkdir -p \"$CAND/node_modules\"\n podman_args=(run --rm --pull=never --read-only\n --tmpfs /tmp:rw,nosuid,nodev,noexec,size=512m\n --cap-drop=ALL\n --security-opt=no-new-privileges\n --userns=keep-id\n --user \"$(id -u):$(id -g)\"\n --network slirp4netns:allow_host_loopback=false\n --workdir /workspace\n --env HOME=/tmp\n --env XDG_CONFIG_HOME=/tmp/config\n --env XDG_CACHE_HOME=/tmp/cache\n --env PI_CODING_AGENT_DIR=/tmp/pi\n --env PI_TELEMETRY=0\n --env \"$secret_name\"\n --mount \"type=bind,src=$CAND,dst=/workspace,rw\"\n --mount \"type=bind,src=$ROOT/fixture/node_modules,dst=/workspace/node_modules,ro\"\n --mount \"type=bind,src=$pi_root,dst=/opt/pi,ro\")\n\n if [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n podman_args+=(--label \"lending-desk.runtime-probe=$BENCH_RUNTIME_PROBE_LABEL\")\n fi\n\n case \"$ARM\" in\n a)\n pi_flags=(--no-skills --no-extensions --no-context-files --no-prompt-templates --no-themes)\n arm_a_agents_backup=\"$OUT/arm-a-AGENTS.md\"\n ;;\n b)\n # Arm B reads the candidate's fixture AGENTS.md and the explicit skills below.\n pi_flags=(--no-extensions --no-prompt-templates --no-themes)\n for skill in measure next-best-practices vercel-react-best-practices vercel-composition-patterns build-graph; do\n [ -f \"$skill_root/$skill/SKILL.md\" ] || { echo \"required Arm B skill unavailable: $skill\" >&2; return 2; }\n podman_args+=(--mount \"type=bind,src=$skill_root/$skill,dst=/opt/skills/$skill,ro\")\n pi_flags+=(--skill \"/opt/skills/$skill\")\n done\n ;;\n *) echo \"arm must be a or b\" >&2; return 2 ;;\n esac\n\n write_agent_identity || { echo \"could not resolve agent execution identity\" >&2; return 2; }\n [ -n \"${!secret_name:-}\" ] || { echo \"missing required provider environment: $secret_name\" >&2; return 2; }\n if [ -n \"$arm_a_agents_backup\" ]; then\n [ -f \"$CAND/AGENTS.md\" ] || { echo \"Arm A candidate AGENTS.md is missing\" >&2; return 2; }\n mv \"$CAND/AGENTS.md\" \"$arm_a_agents_backup\"\n fi\n\n if podman \"${podman_args[@]}\" --entrypoint /usr/bin/timeout \"$pi_image\" \\\n \"${AGENT_TIMEOUT:-3600}\" /opt/pi/bin/pi \\\n --provider \"$PROVIDER\" --model \"$MODEL\" \\\n --print --mode json --no-session --thinking \"$THINKING\" --approve \\\n --tools read,write,edit,bash,grep,find,ls \\\n \"${pi_flags[@]}\" \"$PROMPT\"; then\n agent_rc=0\n else\n agent_rc=$?\n fi\n if [ -n \"$arm_a_agents_backup\" ]; then\n mv \"$arm_a_agents_backup\" \"$CAND/AGENTS.md\" || return 2\n fi\n return \"$agent_rc\"\n}\n\n\n# 0. Host readiness. Checked before the agent runs so a loaded machine costs\n# nothing rather than an hour of tokens followed by void timing results.\n# 6 means \"busy but BENCH_IGNORE_LOAD=1\" — proceed; grade.sh flags the record.\nif \"$H/preflight.sh\" \"$RUN_ID\"; then\n PF=0\nelse\n PF=$?\nfi\n[ \"$PF\" -eq 0 ] || [ \"$PF\" -eq 6 ] || exit 5\n\nrm -rf \"$WORK\"; mkdir -p \"$OUT\"\nif [ \"$RUNTIME_PROBE\" = \"ports\" ]; then\n runtime_port_probe\nfi\n\n# The bounded port probe must publish its allocation promptly. Normal runs\n# still resolve the candidate contract before any fixture or agent work.\nWRITABLE_PATHS=$(\"$H/node_modules/.bin/tsx\" \"$H/candidate-contract.ts\" --writable-paths) || { echo \"candidate contract allowlist is unavailable\" >&2; exit 2; }\nmapfile -t WRITABLE_CANDIDATE_PATHS <<< \"$WRITABLE_PATHS\"\n[ \"${#WRITABLE_CANDIDATE_PATHS[@]}\" -gt 0 ] || { echo \"candidate contract allowlist is empty\" >&2; exit 2; }\nunset WRITABLE_PATHS\n\necho \"[$RUN_ID] $PROVIDER/$MODEL arm=$ARM\"\n\n# 1. Fresh fixture, byte-identical between arms.\nmkdir -p \"$CAND\"\ntar -C \"$ROOT/fixture\" --exclude=node_modules --exclude=.next --exclude=.git \\\n --exclude='*.tsbuildinfo' -cf - . | tar -C \"$CAND\" -xf -\n# Dependencies are mounted read-only for the agent and symlinked only after verification.\n\n\nPROMPT='Read measure/tracks/lending_desk/spec.md and measure/tracks/lending_desk/plan.md, then implement the plan. Mark each plan task complete as you finish it. Run \"npm run typecheck\" and \"npm run build\" and make sure both pass before you stop.'\n\nif [ \"$RUNTIME_PROBE\" = \"container\" ]; then\n echo \"[$RUN_ID] running isolated container runtime probe\"\n if run_container_probe > \"$OUT/session.json\" 2> \"$OUT/session.err\"; then\n exit 0\n else\n PROBE_RC=$?\n exit \"$PROBE_RC\"\n fi\nfi\n\nif [ \"$RUNTIME_PROBE\" = \"dependency-copy\" ]; then\n echo \"[$RUN_ID] running bounded dependency-copy probe\"\n if hydrate_candidate_dependencies; then\n exit 0\n else\n echo \"[$RUN_ID] dependency-copy probe could not create a private copy\" >&2\n exit 2\n fi\nfi\n# 2. Agent.\necho \"[$RUN_ID] invoking agent...\"\nSTART=$(date +%s)\nif [ \"${AGENT_SKIP:-0}\" = \"1\" ]; then\n RUN_MODE=\"overlay\"; AGENT_ISOLATED=false\n if ! apply_overlay \"${OVERLAY:-$ROOT/reference}\"; then\n write_invalid \"overlay could not be applied\"\n exit 3\n fi\n if ! write_overlay_identity; then\n write_invalid \"overlay execution identity could not be resolved\"\n exit 3\n fi\n AGENT_EXIT=0\n echo \"[$RUN_ID] AGENT_SKIP=1 - overlay ${OVERLAY:-$ROOT/reference} applied\"\nelse\n RUN_MODE=\"agent\"; AGENT_ISOLATED=true\n if run_agent_container > \"$OUT/session.json\" 2> \"$OUT/session.err\"; then\n AGENT_EXIT=0\n else\n AGENT_EXIT=$?\n fi\nfi\nEND=$(date +%s)\necho \"[$RUN_ID] agent exit=$AGENT_EXIT in $((END-START))s\"\n\nif ! assert_candidate_contract; then\n echo \"[$RUN_ID] ABORT: candidate violates the filesystem contract\" >&2\n write_invalid \"candidate violates the filesystem contract\"\n exit 3\nfi\nif ! write_provenance; then\n echo \"[$RUN_ID] ABORT: candidate provenance could not be written\" >&2\n write_invalid \"candidate provenance could not be written\"\n exit 3\nfi\n\ndiff -ru \"$ROOT/fixture\" \"$CAND\" -x node_modules -x .next -x '*.tsbuildinfo' \\\n > \"$OUT/candidate.diff\" 2>/dev/null || true\n\n# 2b. Did the agent actually run? `pi` exits 0 on a provider error (no API key,\n# lapsed token, refused request, unknown model id), which used to produce a graded\n# score for an untouched fixture.\n#\n# Two signals, both about work actually produced:\n# - empty diff (a-gpt-5.6-luna, 2026-07-31: lapsed OAuth token)\n# - no output tokens (a-ling-3.0-flash-r1, 2026-08-01: an unresolvable model id\n# returns one empty turn. Its diff was 1 line, so `! -s` did\n# not fire, the gate went green, and it scored 3.1 for $0.)\n#\n# Do NOT add a check on pi's \"not found for provider ... using custom model id\"\n# warning. It is emitted for any id outside pi's local registry and says nothing\n# about whether the call succeeds: on 2026-08-01 both inclusionai/ling-3.0-flash\n# (0 tokens, dead) and inclusionai/ling-3.0-flash:free (219MB transcript, 23KB diff,\n# a real submission) printed it verbatim. Grepping for it aborted the working run.\nif [ \"${AGENT_SKIP:-0}\" != \"1\" ]; then\n INVALID=\"\"\n [ -s \"$OUT/candidate.diff\" ] || INVALID=\"agent produced no changes\"\n OUT_TOKENS=$(\"$H/node_modules/.bin/tsx\" \"$H/usage.ts\" \"$OUT/session.json\" 2>/dev/null \\\n | python3 -c \"import json,sys; print(json.load(sys.stdin).get('output',0))\" 2>/dev/null || echo 0)\n [ \"${OUT_TOKENS:-0}\" -gt 0 ] 2>/dev/null || INVALID=\"agent emitted no output tokens\"\n\n if [ -n \"$INVALID\" ]; then\n echo \"[$RUN_ID] ABORT: $INVALID\" >&2\n { grep -o '\"errorMessage\":\"[^\"]*\"' \"$OUT/session.json\" 2>/dev/null | sort -u | head -3 >&2; } || true\n tail -3 \"$OUT/session.err\" >&2 2>/dev/null || true\n write_invalid \"$INVALID\"\n exit 3\n fi\nfi\n\nif ! hydrate_candidate_dependencies; then\n echo \"[$RUN_ID] ABORT: unable to create private candidate dependencies\" >&2\n write_invalid \"unable to create private candidate dependencies\"\n exit 3\nfi\n\n# here is what let it drift out of sync with collect.ts.\nif AGENT_EXIT=\"$AGENT_EXIT\" THINKING=\"$THINKING\" API_PORT=\"$API_PORT\" E2E_PORT=\"$E2E_PORT\" \\\n \"$H/grade.sh\" \"$RUN_ID\" \"$PROVIDER/$MODEL\" \"$ARM\" \"$((END-START))\"; then\n GRADE_RC=0\nelse\n GRADE_RC=$?\nfi\n\n# 6. Compress the transcript. pi repeats the whole message on every message_update,\n# so these reach ~1GB each (a-deepseek-v4-flash: 939MB) and the matrix runs every\n# model ten times. Done after scoring, since extractUsage reads it; usage.ts accepts\n# the .gz form so a later standalone regrade still reports usage.\nif [ -s \"$OUT/session.json\" ]; then\n gzip -f \"$OUT/session.json\" && echo \"[$RUN_ID] transcript compressed -> session.json.gz\"\nfi\nexit $GRADE_RC\n"}}}} |