1 line
85 KiB
JSON
1 line
85 KiB
JSON
{"repo": "ch1nyzzz/optimizer1", "files": {"scripts/launch_terminus_codex_azure_daytona.sh": {"8624ef6a03f3e1e6fff81c2dbbee15e39bf38383": {"t": 1785745953.720185, "n": 12540, "h": "27e9eaf8f7997efe", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n# * proposer = Codex CLI authenticated against Azure OpenAI (default), or the\n# Claude Code CLI routed at a provider's anthropic-compatible\n# endpoint. Pick with PROPOSER_AGENT=codex|claude — see the\n# proposer knobs below and docs/TERMINUS_CODEX_AZURE.md.\n# * solver = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n# SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n# official DeepSeek API, not Together AI.\n# * rollout = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n# bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n# * default arm: --selection-policy default\n# (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n# * organized arm: --organized --selection-policy default\n# (upstream-2 summaries, skill mode \"organized-summaries\",\n# generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\n# DAYTONA_API_KEY is always required (every rollout is a Daytona sandbox). The\n# proposer's own secret check is PROPOSER_AGENT-dependent and runs after the\n# knobs block below.\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n echo \" a managed-cloud key from https://app.daytona.io.\" >&2\n exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# ---- proposer (PROPOSER_AGENT=codex|claude) -----------------------------\nPROPOSER_AGENT=\"${PROPOSER_AGENT:-codex}\"\n\n# Codex proposer (Azure OpenAI) — used when PROPOSER_AGENT=codex. CODEX_MODEL\n# MUST be your Azure *deployment* name — it is forwarded as `codex exec -m`,\n# and config.toml's model_provider routes it to Azure. Keep it in sync with\n# `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\" # empty => ~/.codex (must hold the Azure config.toml)\n\n# Claude Code proposer — used when PROPOSER_AGENT=claude. Azure OpenAI is NOT\n# anthropic-compatible, so the Claude proposer is routed at a provider's\n# anthropic-compatible endpoint instead. Configure your provider:\n# CLAUDE_BASE_URL the provider's anthropic-compatible base URL\n# CLAUDE_MODEL the model id that provider expects\n# CLAUDE_API_KEY_ENV names the .env variable holding the provider API key\n# CLAUDE_EFFORT optional thinking effort (low|medium|high|xhigh|max)\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-}\"\nCLAUDE_API_KEY_ENV=\"${CLAUDE_API_KEY_ENV:-ANTHROPIC_AUTH_TOKEN}\"\nCLAUDE_EFFORT=\"${CLAUDE_EFFORT:-}\"\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to. SOLVER_MODEL is a litellm\n# <provider>/<model> id — set it to match how your DeepSeek-V4-Pro\n# provider/endpoint is configured.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n echo \" unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\n# ---- preflight: proposer ------------------------------------------------\n# Build the proposer CLI fragment and check the secret the chosen agent needs.\nproposer_args=()\nif [ \"$PROPOSER_AGENT\" = \"codex\" ]; then\n if [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n echo \" against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\n fi\n proposer_args=(\n --proposer-agent codex\n --codex-model \"$CODEX_MODEL\"\n --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n )\n [ -n \"$CODEX_HOME\" ] && proposer_args+=(--codex-home \"$CODEX_HOME\")\nelif [ \"$PROPOSER_AGENT\" = \"claude\" ]; then\n claude_key=\"${!CLAUDE_API_KEY_ENV:-}\"\n if [ -z \"$CLAUDE_BASE_URL\" ] || [ -z \"$CLAUDE_MODEL\" ]; then\n echo \"error: PROPOSER_AGENT=claude needs CLAUDE_BASE_URL and CLAUDE_MODEL set\" >&2\n echo \" to your provider's anthropic-compatible endpoint and model id —\" >&2\n echo \" see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\n fi\n if [ -z \"$claude_key\" ]; then\n echo \"error: \\$$CLAUDE_API_KEY_ENV is empty. PROPOSER_AGENT=claude reads the\" >&2\n echo \" provider API key from the .env variable named by CLAUDE_API_KEY_ENV.\" >&2\n exit 1\n fi\n if ! command -v claude >/dev/null 2>&1; then\n echo \"error: the 'claude' CLI is not on PATH. PROPOSER_AGENT=claude runs the\" >&2\n echo \" Claude Code CLI — install it: npm install -g @anthropic-ai/claude-code\" >&2\n exit 1\n fi\n proposer_args=(\n --proposer-agent claude\n --claude-base-url \"$CLAUDE_BASE_URL\"\n --claude-model \"$CLAUDE_MODEL\"\n --claude-auth-token \"$claude_key\"\n )\n [ -n \"$CLAUDE_EFFORT\" ] && proposer_args+=(--claude-effort \"$CLAUDE_EFFORT\")\nelse\n echo \"error: PROPOSER_AGENT must be 'codex' or 'claude'; got '$PROPOSER_AGENT'\" >&2\n exit 1\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s proposer=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n \"$(date -Is)\" \"$TS\" \"$PROPOSER_AGENT\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + the chosen proposer (proposer_args, built\n# in the preflight above). The per-run bits (--run-id, --iterations, arm flags,\n# test-frontier) are added by the callers.\ncommon_args=(\n --terminus\n --split \"$SPLIT\"\n --limit \"$LIMIT\"\n --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n --terminus-env daytona\n --terminus-solver-model \"$SOLVER_MODEL\"\n --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n \"${proposer_args[@]}\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n return 0\n fi\n local prime_log=\"logs/${baseline_run_id}.log\"\n printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$baseline_run_id\" \\\n --iterations 0 \\\n --selection-policy default \\\n --no-test-frontier \\\n > \"$prime_log\" 2>&1\n local rc=$?\n if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n return 1\n fi\n printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n local arm=\"$1\"\n local arm_args=() run_id log_path\n if [ \"$arm\" = \"default\" ]; then\n # No summary flag: the proposer gets the upstream-2 summary files and\n # skill mode \"default\" (no RunStore tools). --no-summary would instead\n # withhold the summaries entirely, breaking parity with the organized arm.\n arm_args=(--selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n elif [ \"$arm\" = \"organized\" ]; then\n arm_args=(--organized --selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n else\n printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n return 0\n fi\n\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s baseline=%s\\n[%s] LOG %s\\n' \\\n \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n # No --no-test-frontier: after the 20 iterations the optimizer automatically\n # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n setsid nohup python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --baseline-dir \"$BASELINE_DIR\" \\\n --test-frontier-candidate-limit 1 \\\n \"${arm_args[@]}\" \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n echo \"error: baseline prime failed — see $status_file\" >&2\n exit 1\n}\n\nfor arm in default organized; do\n contains \"$ARMS\" \"$arm\" || continue\n start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "7e699a8a3645dfdb1732855649a6105f26ccfdbe": {"t": 1785758239.7900686, "n": 12540, "h": "27e9eaf8f7997efe", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n# * proposer = Codex CLI authenticated against Azure OpenAI (default), or the\n# Claude Code CLI routed at a provider's anthropic-compatible\n# endpoint. Pick with PROPOSER_AGENT=codex|claude — see the\n# proposer knobs below and docs/TERMINUS_CODEX_AZURE.md.\n# * solver = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n# SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n# official DeepSeek API, not Together AI.\n# * rollout = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n# bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n# * default arm: --selection-policy default\n# (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n# * organized arm: --organized --selection-policy default\n# (upstream-2 summaries, skill mode \"organized-summaries\",\n# generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\n# DAYTONA_API_KEY is always required (every rollout is a Daytona sandbox). The\n# proposer's own secret check is PROPOSER_AGENT-dependent and runs after the\n# knobs block below.\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n echo \" a managed-cloud key from https://app.daytona.io.\" >&2\n exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# ---- proposer (PROPOSER_AGENT=codex|claude) -----------------------------\nPROPOSER_AGENT=\"${PROPOSER_AGENT:-codex}\"\n\n# Codex proposer (Azure OpenAI) — used when PROPOSER_AGENT=codex. CODEX_MODEL\n# MUST be your Azure *deployment* name — it is forwarded as `codex exec -m`,\n# and config.toml's model_provider routes it to Azure. Keep it in sync with\n# `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\" # empty => ~/.codex (must hold the Azure config.toml)\n\n# Claude Code proposer — used when PROPOSER_AGENT=claude. Azure OpenAI is NOT\n# anthropic-compatible, so the Claude proposer is routed at a provider's\n# anthropic-compatible endpoint instead. Configure your provider:\n# CLAUDE_BASE_URL the provider's anthropic-compatible base URL\n# CLAUDE_MODEL the model id that provider expects\n# CLAUDE_API_KEY_ENV names the .env variable holding the provider API key\n# CLAUDE_EFFORT optional thinking effort (low|medium|high|xhigh|max)\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-}\"\nCLAUDE_API_KEY_ENV=\"${CLAUDE_API_KEY_ENV:-ANTHROPIC_AUTH_TOKEN}\"\nCLAUDE_EFFORT=\"${CLAUDE_EFFORT:-}\"\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to. SOLVER_MODEL is a litellm\n# <provider>/<model> id — set it to match how your DeepSeek-V4-Pro\n# provider/endpoint is configured.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n echo \" unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\n# ---- preflight: proposer ------------------------------------------------\n# Build the proposer CLI fragment and check the secret the chosen agent needs.\nproposer_args=()\nif [ \"$PROPOSER_AGENT\" = \"codex\" ]; then\n if [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n echo \" against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\n fi\n proposer_args=(\n --proposer-agent codex\n --codex-model \"$CODEX_MODEL\"\n --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n )\n [ -n \"$CODEX_HOME\" ] && proposer_args+=(--codex-home \"$CODEX_HOME\")\nelif [ \"$PROPOSER_AGENT\" = \"claude\" ]; then\n claude_key=\"${!CLAUDE_API_KEY_ENV:-}\"\n if [ -z \"$CLAUDE_BASE_URL\" ] || [ -z \"$CLAUDE_MODEL\" ]; then\n echo \"error: PROPOSER_AGENT=claude needs CLAUDE_BASE_URL and CLAUDE_MODEL set\" >&2\n echo \" to your provider's anthropic-compatible endpoint and model id —\" >&2\n echo \" see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\n fi\n if [ -z \"$claude_key\" ]; then\n echo \"error: \\$$CLAUDE_API_KEY_ENV is empty. PROPOSER_AGENT=claude reads the\" >&2\n echo \" provider API key from the .env variable named by CLAUDE_API_KEY_ENV.\" >&2\n exit 1\n fi\n if ! command -v claude >/dev/null 2>&1; then\n echo \"error: the 'claude' CLI is not on PATH. PROPOSER_AGENT=claude runs the\" >&2\n echo \" Claude Code CLI — install it: npm install -g @anthropic-ai/claude-code\" >&2\n exit 1\n fi\n proposer_args=(\n --proposer-agent claude\n --claude-base-url \"$CLAUDE_BASE_URL\"\n --claude-model \"$CLAUDE_MODEL\"\n --claude-auth-token \"$claude_key\"\n )\n [ -n \"$CLAUDE_EFFORT\" ] && proposer_args+=(--claude-effort \"$CLAUDE_EFFORT\")\nelse\n echo \"error: PROPOSER_AGENT must be 'codex' or 'claude'; got '$PROPOSER_AGENT'\" >&2\n exit 1\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s proposer=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n \"$(date -Is)\" \"$TS\" \"$PROPOSER_AGENT\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + the chosen proposer (proposer_args, built\n# in the preflight above). The per-run bits (--run-id, --iterations, arm flags,\n# test-frontier) are added by the callers.\ncommon_args=(\n --terminus\n --split \"$SPLIT\"\n --limit \"$LIMIT\"\n --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n --terminus-env daytona\n --terminus-solver-model \"$SOLVER_MODEL\"\n --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n \"${proposer_args[@]}\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n return 0\n fi\n local prime_log=\"logs/${baseline_run_id}.log\"\n printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$baseline_run_id\" \\\n --iterations 0 \\\n --selection-policy default \\\n --no-test-frontier \\\n > \"$prime_log\" 2>&1\n local rc=$?\n if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n return 1\n fi\n printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n local arm=\"$1\"\n local arm_args=() run_id log_path\n if [ \"$arm\" = \"default\" ]; then\n # No summary flag: the proposer gets the upstream-2 summary files and\n # skill mode \"default\" (no RunStore tools). --no-summary would instead\n # withhold the summaries entirely, breaking parity with the organized arm.\n arm_args=(--selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n elif [ \"$arm\" = \"organized\" ]; then\n arm_args=(--organized --selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n else\n printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n return 0\n fi\n\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s baseline=%s\\n[%s] LOG %s\\n' \\\n \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n # No --no-test-frontier: after the 20 iterations the optimizer automatically\n # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n setsid nohup python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --baseline-dir \"$BASELINE_DIR\" \\\n --test-frontier-candidate-limit 1 \\\n \"${arm_args[@]}\" \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n echo \"error: baseline prime failed — see $status_file\" >&2\n exit 1\n}\n\nfor arm in default organized; do\n contains \"$ARMS\" \"$arm\" || continue\n start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "75db5644133af8ced48316ce53738341568665c9": {"t": 1785758242.0079272, "n": 9907, "h": "acbc27cf71846bc9", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n# * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n# key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n# there is no interactive Codex \"login\" for Azure.\n# * solver = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n# SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n# official DeepSeek API, not Together AI.\n# * rollout = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n# bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n# * default arm: --selection-policy default\n# (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n# * organized arm: --organized --selection-policy default\n# (upstream-2 summaries, skill mode \"organized-summaries\",\n# generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n echo \" a managed-cloud key from https://app.daytona.io.\" >&2\n exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n echo \" against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\" # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to. SOLVER_MODEL is a litellm\n# <provider>/<model> id — set it to match how your DeepSeek-V4-Pro\n# provider/endpoint is configured.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n echo \" unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n --terminus\n --split \"$SPLIT\"\n --limit \"$LIMIT\"\n --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n --terminus-env daytona\n --terminus-solver-model \"$SOLVER_MODEL\"\n --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n --proposer-agent codex\n --codex-model \"$CODEX_MODEL\"\n --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n return 0\n fi\n local prime_log=\"logs/${baseline_run_id}.log\"\n printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$baseline_run_id\" \\\n --iterations 0 \\\n --selection-policy default \\\n --no-test-frontier \\\n > \"$prime_log\" 2>&1\n local rc=$?\n if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n return 1\n fi\n printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n local arm=\"$1\"\n local arm_args=() run_id log_path\n if [ \"$arm\" = \"default\" ]; then\n # No summary flag: the proposer gets the upstream-2 summary files and\n # skill mode \"default\" (no RunStore tools). --no-summary would instead\n # withhold the summaries entirely, breaking parity with the organized arm.\n arm_args=(--selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n elif [ \"$arm\" = \"organized\" ]; then\n arm_args=(--organized --selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n else\n printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n return 0\n fi\n\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s baseline=%s\\n[%s] LOG %s\\n' \\\n \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n # No --no-test-frontier: after the 20 iterations the optimizer automatically\n # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n setsid nohup python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --baseline-dir \"$BASELINE_DIR\" \\\n --test-frontier-candidate-limit 1 \\\n \"${arm_args[@]}\" \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n echo \"error: baseline prime failed — see $status_file\" >&2\n exit 1\n}\n\nfor arm in default organized; do\n contains \"$ARMS\" \"$arm\" || continue\n start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "8e7c564f6382d5be813c7edb744b66ca05897968": {"t": 1785758243.9969687, "n": 9779, "h": "28dcee4d1c7cca94", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n# * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n# key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n# there is no interactive Codex \"login\" for Azure.\n# * solver = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n# SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n# official DeepSeek API, not Together AI.\n# * rollout = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n# bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n# * default arm: --selection-policy default\n# (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n# * organized arm: --organized --selection-policy default\n# (upstream-2 summaries, skill mode \"organized-summaries\",\n# generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n echo \" a managed-cloud key from https://app.daytona.io.\" >&2\n exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n echo \" against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\" # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n echo \" unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n --terminus\n --split \"$SPLIT\"\n --limit \"$LIMIT\"\n --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n --terminus-env daytona\n --terminus-solver-model \"$SOLVER_MODEL\"\n --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n --proposer-agent codex\n --codex-model \"$CODEX_MODEL\"\n --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n return 0\n fi\n local prime_log=\"logs/${baseline_run_id}.log\"\n printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$baseline_run_id\" \\\n --iterations 0 \\\n --selection-policy default \\\n --no-test-frontier \\\n > \"$prime_log\" 2>&1\n local rc=$?\n if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n return 1\n fi\n printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n local arm=\"$1\"\n local arm_args=() run_id log_path\n if [ \"$arm\" = \"default\" ]; then\n # No summary flag: the proposer gets the upstream-2 summary files and\n # skill mode \"default\" (no RunStore tools). --no-summary would instead\n # withhold the summaries entirely, breaking parity with the organized arm.\n arm_args=(--selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n elif [ \"$arm\" = \"organized\" ]; then\n arm_args=(--organized --selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n else\n printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n return 0\n fi\n\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s baseline=%s\\n[%s] LOG %s\\n' \\\n \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n # No --no-test-frontier: after the 20 iterations the optimizer automatically\n # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n setsid nohup python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --baseline-dir \"$BASELINE_DIR\" \\\n --test-frontier-candidate-limit 1 \\\n \"${arm_args[@]}\" \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n echo \"error: baseline prime failed — see $status_file\" >&2\n exit 1\n}\n\nfor arm in default organized; do\n contains \"$ARMS\" \"$arm\" || continue\n start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "308805cd5f8184ff3e56367f978de84c6a1ac3c1": {"t": 1785758246.4679558, "n": 9767, "h": "7095b9d121d6b983", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n# * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n# key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n# there is no interactive Codex \"login\" for Azure.\n# * solver = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n# SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n# official DeepSeek API, not Together AI.\n# * rollout = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n# bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n# * default arm: --selection-policy default\n# (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n# * organized arm: --organized --selection-policy default\n# (upstream-2 summaries, skill mode \"organized-summaries\",\n# generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n echo \" a managed-cloud key from https://app.daytona.io.\" >&2\n exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n echo \" against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\" # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n echo \" unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n --terminus\n --split \"$SPLIT\"\n --limit \"$LIMIT\"\n --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n --terminus-env daytona\n --terminus-solver-model \"$SOLVER_MODEL\"\n --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n --proposer-agent codex\n --codex-model \"$CODEX_MODEL\"\n --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n if [ -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n return 0\n fi\n local prime_log=\"logs/${baseline_run_id}.log\"\n printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$baseline_run_id\" \\\n --iterations 0 \\\n --selection-policy default \\\n --no-test-frontier \\\n > \"$prime_log\" 2>&1\n local rc=$?\n if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n return 1\n fi\n printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n local arm=\"$1\"\n local arm_args=() run_id log_path\n if [ \"$arm\" = \"default\" ]; then\n # No summary flag: the proposer gets the upstream-2 summary files and\n # skill mode \"default\" (no RunStore tools). --no-summary would instead\n # withhold the summaries entirely, breaking parity with the organized arm.\n arm_args=(--selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n elif [ \"$arm\" = \"organized\" ]; then\n arm_args=(--organized --selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n else\n printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n return 0\n fi\n\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s baseline=%s\\n[%s] LOG %s\\n' \\\n \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n # No --no-test-frontier: after the 20 iterations the optimizer automatically\n # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n setsid nohup python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --baseline-dir \"$BASELINE_DIR\" \\\n --test-frontier-candidate-limit 1 \\\n \"${arm_args[@]}\" \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n echo \"error: baseline prime failed — see $status_file\" >&2\n exit 1\n}\n\nfor arm in default organized; do\n contains \"$ARMS\" \"$arm\" || continue\n start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "c65db10e3af9ad72b5b2c5493eb6f2a877e8bfaf": {"t": 1785758247.7479098, "n": 9288, "h": "b3d0c7d8c6e1a569", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n# * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n# key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n# there is no interactive Codex \"login\" for Azure.\n# * solver = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n# SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n# official DeepSeek API, not Together AI.\n# * rollout = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n# bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface:\n# * default arm: --no-summary --selection-policy default\n# (no summary files, no RunStore tools)\n# * organized arm: --organized --selection-policy default\n# (state.md + RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n echo \" a managed-cloud key from https://app.daytona.io.\" >&2\n exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n echo \" against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\" # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n echo \" unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n --terminus\n --split \"$SPLIT\"\n --limit \"$LIMIT\"\n --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n --terminus-env daytona\n --terminus-solver-model \"$SOLVER_MODEL\"\n --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n --proposer-agent codex\n --codex-model \"$CODEX_MODEL\"\n --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n if [ -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n return 0\n fi\n local prime_log=\"logs/${baseline_run_id}.log\"\n printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$baseline_run_id\" \\\n --iterations 0 \\\n --selection-policy default \\\n --no-test-frontier \\\n > \"$prime_log\" 2>&1\n local rc=$?\n if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n return 1\n fi\n printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n local arm=\"$1\"\n local arm_args=() run_id log_path\n if [ \"$arm\" = \"default\" ]; then\n arm_args=(--no-summary --selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n elif [ \"$arm\" = \"organized\" ]; then\n arm_args=(--organized --selection-policy default)\n run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n else\n printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n return 0\n fi\n\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s baseline=%s\\n[%s] LOG %s\\n' \\\n \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n # No --no-test-frontier: after the 20 iterations the optimizer automatically\n # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n setsid nohup python -m optimizer1.cli optimize \\\n \"${common_args[@]}\" \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --baseline-dir \"$BASELINE_DIR\" \\\n --test-frontier-candidate-limit 1 \\\n \"${arm_args[@]}\" \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n echo \"error: baseline prime failed — see $status_file\" >&2\n exit 1\n}\n\nfor arm in default organized; do\n contains \"$ARMS\" \"$arm\" || continue\n start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}}, "scripts/launch_deepseek_v4_pro_pure_default_notrace.sh": {"8624ef6a03f3e1e6fff81c2dbbee15e39bf38383": {"t": 1785745960.4925592, "n": 3986, "h": "ab14dd6792c05f82", "c": "#!/usr/bin/env bash\n# Launch the pure default no-trace control runs across LoCoMo and\n# LongMemEval-S. This is the baseline arm used to compare against\n# organized default: default selection, clean patch base, no state.md,\n# and no proposer-facing trace/RunStore MCP tools.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\nif [ -z \"${DEEPSEEK_API_KEY:-}\" ]; then\n printf 'fatal: DEEPSEEK_API_KEY is not set; populate .env or export it.\\n' >&2\n exit 2\nfi\n\nunset ANTHROPIC_API_KEY\nexport ANTHROPIC_AUTH_TOKEN=\"${DEEPSEEK_API_KEY}\"\nexport ANTHROPIC_BASE_URL=\"${ANTHROPIC_BASE_URL:-https://api.deepseek.com/anthropic}\"\nexport ANTHROPIC_MODEL=\"${ANTHROPIC_MODEL:-deepseek-v4-pro[1m]}\"\n\nif [ -n \"${TOGETHER_API_KEY:-}\" ]; then\n export OPENAI_API_KEY=\"${TOGETHER_API_KEY}\"\n export OPENAI_BASE_URL=\"https://api.together.xyz/v1\"\nfi\nexport DIFF_EMBEDDING_MODEL=\"${DIFF_EMBEDDING_MODEL:-BAAI/bge-large-en-v1.5}\"\n\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nITERATIONS=\"${ITERATIONS:-50}\"\nTASKS=\"${TASKS:-locomo,longmemeval}\"\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-deepseek-v4-pro[1m]}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-https://api.deepseek.com/anthropic}\"\nEVAL_WORKERS=\"${EVAL_WORKERS:-128}\"\nEVAL_MODEL=\"${EVAL_MODEL:-/data/home/yuhan/model_zoo/Qwen3-8B}\"\nEVAL_BASE_URL=\"${EVAL_BASE_URL:-http://127.0.0.1:8002/v1}\"\nDOCKER_USER_SPEC=\"${DOCKER_USER_SPEC:-$(id -u):$(id -g)}\"\nDOCKER_IMAGE=\"${DOCKER_IMAGE:-docker-claude:latest}\"\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_deepseek_v4_pro_pure_default_notrace_${TS}.status\"\n: > \"$status_file\"\nprintf 'timestamp=%s\\n' \"$TS\" >> \"$status_file\"\nprintf 'mode=pure_default_no_trace_harness\\n' >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\nstart_one() {\n local task=\"$1\"\n local task_label task_args=() run_id log_path\n\n if [ \"$task\" = \"locomo\" ]; then\n task_label=\"locomo\"\n task_args=(--locomo)\n elif [ \"$task\" = \"longmemeval\" ]; then\n task_label=\"longmemeval_s\"\n task_args=(--longmemeval --longmemeval-variant s)\n else\n printf '[%s] SKIP unknown_task=%s\\n' \"$(date -Is)\" \"$task\" >> \"$status_file\"\n return 0\n fi\n\n run_id=\"${task_label}_deepseek_v4_pro_pure_default_notrace_iter${ITERATIONS}_${TS}\"\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n setsid python -m optimizer1.cli optimize \\\n \"${task_args[@]}\" \\\n --selection-policy default \\\n --proposer-no-trace-harness-section \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --split train \\\n --scaffolds memgpt_source \\\n --scaffold-extra-json @configs/source_memory.example.json \\\n --eval-workers \"$EVAL_WORKERS\" \\\n --model \"$EVAL_MODEL\" \\\n --base-url \"$EVAL_BASE_URL\" \\\n --proposer-agent claude \\\n --claude-model \"$CLAUDE_MODEL\" \\\n --claude-base-url \"$CLAUDE_BASE_URL\" \\\n --claude-effort max \\\n --force-budget high \\\n --proposer-sandbox docker \\\n --proposer-docker-image \"$DOCKER_IMAGE\" \\\n --proposer-docker-user \"$DOCKER_USER_SPEC\" \\\n --proposer-docker-home /tmp \\\n --proposer-docker-env DEEPSEEK_API_KEY \\\n --proposer-docker-env ANTHROPIC_AUTH_TOKEN \\\n --proposer-docker-env ANTHROPIC_BASE_URL \\\n --proposer-docker-env ANTHROPIC_MODEL \\\n --proposer-docker-env OPENAI_API_KEY \\\n --proposer-docker-env OPENAI_BASE_URL \\\n --proposer-docker-env DIFF_EMBEDDING_MODEL \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\nfor task in locomo longmemeval; do\n contains \"$TASKS\" \"$task\" || continue\n start_one \"$task\"\ndone\n\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "5cefec0bc430e944c304d1ae937b4defd0578a1f": {"t": 1785758237.4609592, "n": 3986, "h": "ab14dd6792c05f82", "c": "#!/usr/bin/env bash\n# Launch the pure default no-trace control runs across LoCoMo and\n# LongMemEval-S. This is the baseline arm used to compare against\n# organized default: default selection, clean patch base, no state.md,\n# and no proposer-facing trace/RunStore MCP tools.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\nif [ -z \"${DEEPSEEK_API_KEY:-}\" ]; then\n printf 'fatal: DEEPSEEK_API_KEY is not set; populate .env or export it.\\n' >&2\n exit 2\nfi\n\nunset ANTHROPIC_API_KEY\nexport ANTHROPIC_AUTH_TOKEN=\"${DEEPSEEK_API_KEY}\"\nexport ANTHROPIC_BASE_URL=\"${ANTHROPIC_BASE_URL:-https://api.deepseek.com/anthropic}\"\nexport ANTHROPIC_MODEL=\"${ANTHROPIC_MODEL:-deepseek-v4-pro[1m]}\"\n\nif [ -n \"${TOGETHER_API_KEY:-}\" ]; then\n export OPENAI_API_KEY=\"${TOGETHER_API_KEY}\"\n export OPENAI_BASE_URL=\"https://api.together.xyz/v1\"\nfi\nexport DIFF_EMBEDDING_MODEL=\"${DIFF_EMBEDDING_MODEL:-BAAI/bge-large-en-v1.5}\"\n\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nITERATIONS=\"${ITERATIONS:-50}\"\nTASKS=\"${TASKS:-locomo,longmemeval}\"\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-deepseek-v4-pro[1m]}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-https://api.deepseek.com/anthropic}\"\nEVAL_WORKERS=\"${EVAL_WORKERS:-128}\"\nEVAL_MODEL=\"${EVAL_MODEL:-/data/home/yuhan/model_zoo/Qwen3-8B}\"\nEVAL_BASE_URL=\"${EVAL_BASE_URL:-http://127.0.0.1:8002/v1}\"\nDOCKER_USER_SPEC=\"${DOCKER_USER_SPEC:-$(id -u):$(id -g)}\"\nDOCKER_IMAGE=\"${DOCKER_IMAGE:-docker-claude:latest}\"\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_deepseek_v4_pro_pure_default_notrace_${TS}.status\"\n: > \"$status_file\"\nprintf 'timestamp=%s\\n' \"$TS\" >> \"$status_file\"\nprintf 'mode=pure_default_no_trace_harness\\n' >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\nstart_one() {\n local task=\"$1\"\n local task_label task_args=() run_id log_path\n\n if [ \"$task\" = \"locomo\" ]; then\n task_label=\"locomo\"\n task_args=(--locomo)\n elif [ \"$task\" = \"longmemeval\" ]; then\n task_label=\"longmemeval_s\"\n task_args=(--longmemeval --longmemeval-variant s)\n else\n printf '[%s] SKIP unknown_task=%s\\n' \"$(date -Is)\" \"$task\" >> \"$status_file\"\n return 0\n fi\n\n run_id=\"${task_label}_deepseek_v4_pro_pure_default_notrace_iter${ITERATIONS}_${TS}\"\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n return 0\n fi\n\n log_path=\"logs/${run_id}.log\"\n printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n setsid python -m optimizer1.cli optimize \\\n \"${task_args[@]}\" \\\n --selection-policy default \\\n --proposer-no-trace-harness-section \\\n --run-id \"$run_id\" \\\n --iterations \"$ITERATIONS\" \\\n --split train \\\n --scaffolds memgpt_source \\\n --scaffold-extra-json @configs/source_memory.example.json \\\n --eval-workers \"$EVAL_WORKERS\" \\\n --model \"$EVAL_MODEL\" \\\n --base-url \"$EVAL_BASE_URL\" \\\n --proposer-agent claude \\\n --claude-model \"$CLAUDE_MODEL\" \\\n --claude-base-url \"$CLAUDE_BASE_URL\" \\\n --claude-effort max \\\n --force-budget high \\\n --proposer-sandbox docker \\\n --proposer-docker-image \"$DOCKER_IMAGE\" \\\n --proposer-docker-user \"$DOCKER_USER_SPEC\" \\\n --proposer-docker-home /tmp \\\n --proposer-docker-env DEEPSEEK_API_KEY \\\n --proposer-docker-env ANTHROPIC_AUTH_TOKEN \\\n --proposer-docker-env ANTHROPIC_BASE_URL \\\n --proposer-docker-env ANTHROPIC_MODEL \\\n --proposer-docker-env OPENAI_API_KEY \\\n --proposer-docker-env OPENAI_BASE_URL \\\n --proposer-docker-env DIFF_EMBEDDING_MODEL \\\n > \"$log_path\" 2>&1 < /dev/null &\n\n local pid=$!\n printf '[%s] PID %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\nfor task in locomo longmemeval; do\n contains \"$TASKS\" \"$task\" || continue\n start_one \"$task\"\ndone\n\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}}, "scripts/launch_claudekimi_swebench_random_recent.sh": {"8624ef6a03f3e1e6fff81c2dbbee15e39bf38383": {"t": 1785745962.573957, "n": 3177, "h": "42755dde9110b3b3", "c": "#!/usr/bin/env bash\nset -u -o pipefail\n\ncd \"$(dirname \"$0\")/..\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\ntimestamp=\"${1:-$(date +%Y%m%d_%H%M%S)}\"\nmax_parallel=\"${MAX_PARALLEL:-3}\"\ndriver_status=\"logs/claudekimi_swebench_random_recent_${timestamp}.status\"\nmkdir -p logs runs\n: > \"$driver_status\"\n\nrun_job() {\n local run_id=\"$1\"\n shift\n local log_path=\"logs/${run_id}.log\"\n {\n printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\"\n printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\"\n } >> \"$driver_status\"\n \"$@\" > \"$log_path\" 2>&1\n local status=$?\n printf '[%s] END %s status=%s\\n' \"$(date -Is)\" \"$run_id\" \"$status\" >> \"$driver_status\"\n return \"$status\"\n}\n\nwait_for_slot() {\n local pattern=\"python -m optimizer1.cli optimize .*claudekimi_.*(3_fileaccess|bandit_v4_autobudget).*_${timestamp}\"\n while [ \"$(pgrep -af \"$pattern\" | wc -l)\" -ge \"$max_parallel\" ]; do\n sleep 30\n done\n}\n\nstart_job() {\n local run_id=\"$1\"\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir=runs/%s\\n' \"$(date -Is)\" \"$run_id\" \"$run_id\" >> \"$driver_status\"\n return 0\n fi\n wait_for_slot\n run_job \"$@\" &\n pids+=(\"$!\")\n}\n\npids=()\n\nminiswe_run_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py run --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir} --model openai/deepseek-v4-flash --base-url https://api.deepseek.com/v1 --max-tokens 4096 --api-key-env DEEPSEEK_API_KEY\"\nminiswe_eval_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py eval --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir}\"\n\nminiswe_common=(\n python -m optimizer1.cli optimize\n --swebench\n --iterations 20\n --split train\n --limit 30\n --swebench-data-path data/swebench_verified_full.json\n --eval-timeout-s 900\n --eval-workers 10\n --proposer-agent kimi\n --proposer-sandbox docker\n --proposer-docker-image docker-claude-kimi:latest\n --proposer-docker-env KIMI_API_KEY\n --proposer-docker-user 1023:1023\n --proposer-docker-home /tmp\n --mini-swe-agent-command \"$miniswe_run_command\"\n --mini-swe-agent-eval-command \"$miniswe_eval_command\"\n)\n\nfor policy in random recent; do\n run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_${policy}3_fileaccess_docker_iter20_trainfirst30_w10_t900_${timestamp}\"\n start_job \"$run_id\" \"${miniswe_common[@]}\" \\\n --run-id \"$run_id\" \\\n --selection-policy \"$policy\"\ndone\n\n# bandit_v4 baseline on the same swebench config (current branch is bandit-v4,\n# so --selection-policy bandit dispatches to the v4 ref-iter selection rule).\nbandit_v4_run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_bandit_v4_autobudget_docker_iter20_trainfirst30_w10_t900_bw16_${timestamp}\"\nstart_job \"$bandit_v4_run_id\" \"${miniswe_common[@]}\" \\\n --run-id \"$bandit_v4_run_id\" \\\n --selection-policy bandit \\\n --bandit-reward-window 16\n\nfailures=0\nfor pid in \"${pids[@]}\"; do\n if ! wait \"$pid\"; then\n failures=$((failures + 1))\n fi\ndone\n\nprintf '[%s] COMPLETE failures=%s\\n' \"$(date -Is)\" \"$failures\" >> \"$driver_status\"\nexit \"$failures\"\n"}, "4466d9fe8d3d818e8bfb0759e520ccbbfa3c4296": {"t": 1785758238.607434, "n": 3177, "h": "42755dde9110b3b3", "c": "#!/usr/bin/env bash\nset -u -o pipefail\n\ncd \"$(dirname \"$0\")/..\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\ntimestamp=\"${1:-$(date +%Y%m%d_%H%M%S)}\"\nmax_parallel=\"${MAX_PARALLEL:-3}\"\ndriver_status=\"logs/claudekimi_swebench_random_recent_${timestamp}.status\"\nmkdir -p logs runs\n: > \"$driver_status\"\n\nrun_job() {\n local run_id=\"$1\"\n shift\n local log_path=\"logs/${run_id}.log\"\n {\n printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\"\n printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\"\n } >> \"$driver_status\"\n \"$@\" > \"$log_path\" 2>&1\n local status=$?\n printf '[%s] END %s status=%s\\n' \"$(date -Is)\" \"$run_id\" \"$status\" >> \"$driver_status\"\n return \"$status\"\n}\n\nwait_for_slot() {\n local pattern=\"python -m optimizer1.cli optimize .*claudekimi_.*(3_fileaccess|bandit_v4_autobudget).*_${timestamp}\"\n while [ \"$(pgrep -af \"$pattern\" | wc -l)\" -ge \"$max_parallel\" ]; do\n sleep 30\n done\n}\n\nstart_job() {\n local run_id=\"$1\"\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir=runs/%s\\n' \"$(date -Is)\" \"$run_id\" \"$run_id\" >> \"$driver_status\"\n return 0\n fi\n wait_for_slot\n run_job \"$@\" &\n pids+=(\"$!\")\n}\n\npids=()\n\nminiswe_run_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py run --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir} --model openai/deepseek-v4-flash --base-url https://api.deepseek.com/v1 --max-tokens 4096 --api-key-env DEEPSEEK_API_KEY\"\nminiswe_eval_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py eval --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir}\"\n\nminiswe_common=(\n python -m optimizer1.cli optimize\n --swebench\n --iterations 20\n --split train\n --limit 30\n --swebench-data-path data/swebench_verified_full.json\n --eval-timeout-s 900\n --eval-workers 10\n --proposer-agent kimi\n --proposer-sandbox docker\n --proposer-docker-image docker-claude-kimi:latest\n --proposer-docker-env KIMI_API_KEY\n --proposer-docker-user 1023:1023\n --proposer-docker-home /tmp\n --mini-swe-agent-command \"$miniswe_run_command\"\n --mini-swe-agent-eval-command \"$miniswe_eval_command\"\n)\n\nfor policy in random recent; do\n run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_${policy}3_fileaccess_docker_iter20_trainfirst30_w10_t900_${timestamp}\"\n start_job \"$run_id\" \"${miniswe_common[@]}\" \\\n --run-id \"$run_id\" \\\n --selection-policy \"$policy\"\ndone\n\n# bandit_v4 baseline on the same swebench config (current branch is bandit-v4,\n# so --selection-policy bandit dispatches to the v4 ref-iter selection rule).\nbandit_v4_run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_bandit_v4_autobudget_docker_iter20_trainfirst30_w10_t900_bw16_${timestamp}\"\nstart_job \"$bandit_v4_run_id\" \"${miniswe_common[@]}\" \\\n --run-id \"$bandit_v4_run_id\" \\\n --selection-policy bandit \\\n --bandit-reward-window 16\n\nfailures=0\nfor pid in \"${pids[@]}\"; do\n if ! wait \"$pid\"; then\n failures=$((failures + 1))\n fi\ndone\n\nprintf '[%s] COMPLETE failures=%s\\n' \"$(date -Is)\" \"$failures\" >> \"$driver_status\"\nexit \"$failures\"\n"}, "2ef73500ed812f64de46f0d6c34e4993a875aef3": {"t": 1785758240.5602186, "n": 3173, "h": "b11760d12df08dfa", "c": "#!/usr/bin/env bash\nset -u -o pipefail\n\ncd \"$(dirname \"$0\")/..\"\n\nif [ -f .env ]; then\n set -a\n # shellcheck disable=SC1091\n source .env\n set +a\nfi\n\ntimestamp=\"${1:-$(date +%Y%m%d_%H%M%S)}\"\nmax_parallel=\"${MAX_PARALLEL:-3}\"\ndriver_status=\"logs/claudekimi_swebench_random_recent_${timestamp}.status\"\nmkdir -p logs runs\n: > \"$driver_status\"\n\nrun_job() {\n local run_id=\"$1\"\n shift\n local log_path=\"logs/${run_id}.log\"\n {\n printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\"\n printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\"\n } >> \"$driver_status\"\n \"$@\" > \"$log_path\" 2>&1\n local status=$?\n printf '[%s] END %s status=%s\\n' \"$(date -Is)\" \"$run_id\" \"$status\" >> \"$driver_status\"\n return \"$status\"\n}\n\nwait_for_slot() {\n local pattern=\"python -m memomemo.cli optimize .*claudekimi_.*(3_fileaccess|bandit_v4_autobudget).*_${timestamp}\"\n while [ \"$(pgrep -af \"$pattern\" | wc -l)\" -ge \"$max_parallel\" ]; do\n sleep 30\n done\n}\n\nstart_job() {\n local run_id=\"$1\"\n if [ -d \"runs/${run_id}\" ]; then\n printf '[%s] SKIP %s existing_run_dir=runs/%s\\n' \"$(date -Is)\" \"$run_id\" \"$run_id\" >> \"$driver_status\"\n return 0\n fi\n wait_for_slot\n run_job \"$@\" &\n pids+=(\"$!\")\n}\n\npids=()\n\nminiswe_run_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py run --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir} --model openai/deepseek-v4-flash --base-url https://api.deepseek.com/v1 --max-tokens 4096 --api-key-env DEEPSEEK_API_KEY\"\nminiswe_eval_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py eval --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir}\"\n\nminiswe_common=(\n python -m memomemo.cli optimize\n --swebench\n --iterations 20\n --split train\n --limit 30\n --swebench-data-path data/swebench_verified_full.json\n --eval-timeout-s 900\n --eval-workers 10\n --proposer-agent kimi\n --proposer-sandbox docker\n --proposer-docker-image docker-claude-kimi:latest\n --proposer-docker-env KIMI_API_KEY\n --proposer-docker-user 1023:1023\n --proposer-docker-home /tmp\n --mini-swe-agent-command \"$miniswe_run_command\"\n --mini-swe-agent-eval-command \"$miniswe_eval_command\"\n)\n\nfor policy in random recent; do\n run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_${policy}3_fileaccess_docker_iter20_trainfirst30_w10_t900_${timestamp}\"\n start_job \"$run_id\" \"${miniswe_common[@]}\" \\\n --run-id \"$run_id\" \\\n --selection-policy \"$policy\"\ndone\n\n# bandit_v4 baseline on the same swebench config (current branch is bandit-v4,\n# so --selection-policy bandit dispatches to the v4 ref-iter selection rule).\nbandit_v4_run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_bandit_v4_autobudget_docker_iter20_trainfirst30_w10_t900_bw16_${timestamp}\"\nstart_job \"$bandit_v4_run_id\" \"${miniswe_common[@]}\" \\\n --run-id \"$bandit_v4_run_id\" \\\n --selection-policy bandit \\\n --bandit-reward-window 16\n\nfailures=0\nfor pid in \"${pids[@]}\"; do\n if ! wait \"$pid\"; then\n failures=$((failures + 1))\n fi\ndone\n\nprintf '[%s] COMPLETE failures=%s\\n' \"$(date -Is)\" \"$failures\" >> \"$driver_status\"\nexit \"$failures\"\n"}}}} |