hack/tools/scripts/llm-key-hunter/results/_cache/content/ch1nyzzz_optimizer1.json

{"repo": "ch1nyzzz/optimizer1", "files": {"scripts/launch_terminus_codex_azure_daytona.sh": {"8624ef6a03f3e1e6fff81c2dbbee15e39bf38383": {"t": 1785745953.720185, "n": 12540, "h": "27e9eaf8f7997efe", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n#   * proposer = Codex CLI authenticated against Azure OpenAI (default), or the\n#                Claude Code CLI routed at a provider's anthropic-compatible\n#                endpoint. Pick with PROPOSER_AGENT=codex|claude — see the\n#                proposer knobs below and docs/TERMINUS_CODEX_AZURE.md.\n#   * solver   = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n#                SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n#                official DeepSeek API, not Together AI.\n#   * rollout  = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n#                bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n#   * default   arm: --selection-policy default\n#                    (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n#   * organized arm: --organized --selection-policy default\n#                    (upstream-2 summaries, skill mode \"organized-summaries\",\n#                     generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\n# DAYTONA_API_KEY is always required (every rollout is a Daytona sandbox). The\n# proposer's own secret check is PROPOSER_AGENT-dependent and runs after the\n# knobs block below.\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n  echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n  echo \"       a managed-cloud key from https://app.daytona.io.\" >&2\n  exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# ---- proposer (PROPOSER_AGENT=codex|claude) -----------------------------\nPROPOSER_AGENT=\"${PROPOSER_AGENT:-codex}\"\n\n# Codex proposer (Azure OpenAI) — used when PROPOSER_AGENT=codex. CODEX_MODEL\n# MUST be your Azure *deployment* name — it is forwarded as `codex exec -m`,\n# and config.toml's model_provider routes it to Azure. Keep it in sync with\n# `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\"   # empty => ~/.codex (must hold the Azure config.toml)\n\n# Claude Code proposer — used when PROPOSER_AGENT=claude. Azure OpenAI is NOT\n# anthropic-compatible, so the Claude proposer is routed at a provider's\n# anthropic-compatible endpoint instead. Configure your provider:\n#   CLAUDE_BASE_URL    the provider's anthropic-compatible base URL\n#   CLAUDE_MODEL       the model id that provider expects\n#   CLAUDE_API_KEY_ENV names the .env variable holding the provider API key\n#   CLAUDE_EFFORT      optional thinking effort (low|medium|high|xhigh|max)\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-}\"\nCLAUDE_API_KEY_ENV=\"${CLAUDE_API_KEY_ENV:-ANTHROPIC_AUTH_TOKEN}\"\nCLAUDE_EFFORT=\"${CLAUDE_EFFORT:-}\"\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to. SOLVER_MODEL is a litellm\n# <provider>/<model> id — set it to match how your DeepSeek-V4-Pro\n# provider/endpoint is configured.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n  echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n  echo \"         unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\n# ---- preflight: proposer ------------------------------------------------\n# Build the proposer CLI fragment and check the secret the chosen agent needs.\nproposer_args=()\nif [ \"$PROPOSER_AGENT\" = \"codex\" ]; then\n  if [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n    echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n    echo \"       against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n    exit 1\n  fi\n  proposer_args=(\n    --proposer-agent codex\n    --codex-model \"$CODEX_MODEL\"\n    --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n  )\n  [ -n \"$CODEX_HOME\" ] && proposer_args+=(--codex-home \"$CODEX_HOME\")\nelif [ \"$PROPOSER_AGENT\" = \"claude\" ]; then\n  claude_key=\"${!CLAUDE_API_KEY_ENV:-}\"\n  if [ -z \"$CLAUDE_BASE_URL\" ] || [ -z \"$CLAUDE_MODEL\" ]; then\n    echo \"error: PROPOSER_AGENT=claude needs CLAUDE_BASE_URL and CLAUDE_MODEL set\" >&2\n    echo \"       to your provider's anthropic-compatible endpoint and model id —\" >&2\n    echo \"       see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n    exit 1\n  fi\n  if [ -z \"$claude_key\" ]; then\n    echo \"error: \\$$CLAUDE_API_KEY_ENV is empty. PROPOSER_AGENT=claude reads the\" >&2\n    echo \"       provider API key from the .env variable named by CLAUDE_API_KEY_ENV.\" >&2\n    exit 1\n  fi\n  if ! command -v claude >/dev/null 2>&1; then\n    echo \"error: the 'claude' CLI is not on PATH. PROPOSER_AGENT=claude runs the\" >&2\n    echo \"       Claude Code CLI — install it: npm install -g @anthropic-ai/claude-code\" >&2\n    exit 1\n  fi\n  proposer_args=(\n    --proposer-agent claude\n    --claude-base-url \"$CLAUDE_BASE_URL\"\n    --claude-model \"$CLAUDE_MODEL\"\n    --claude-auth-token \"$claude_key\"\n  )\n  [ -n \"$CLAUDE_EFFORT\" ] && proposer_args+=(--claude-effort \"$CLAUDE_EFFORT\")\nelse\n  echo \"error: PROPOSER_AGENT must be 'codex' or 'claude'; got '$PROPOSER_AGENT'\" >&2\n  exit 1\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s proposer=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n  \"$(date -Is)\" \"$TS\" \"$PROPOSER_AGENT\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n  >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + the chosen proposer (proposer_args, built\n# in the preflight above). The per-run bits (--run-id, --iterations, arm flags,\n# test-frontier) are added by the callers.\ncommon_args=(\n  --terminus\n  --split \"$SPLIT\"\n  --limit \"$LIMIT\"\n  --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n  --terminus-env daytona\n  --terminus-solver-model \"$SOLVER_MODEL\"\n  --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n  --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n  --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n  --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n  --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n  --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n  \"${proposer_args[@]}\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n  if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n    return 0\n  fi\n  local prime_log=\"logs/${baseline_run_id}.log\"\n  printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n    \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n  python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$baseline_run_id\" \\\n    --iterations 0 \\\n    --selection-policy default \\\n    --no-test-frontier \\\n    > \"$prime_log\" 2>&1\n  local rc=$?\n  if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n      \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n    return 1\n  fi\n  printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n  local arm=\"$1\"\n  local arm_args=() run_id log_path\n  if [ \"$arm\" = \"default\" ]; then\n    # No summary flag: the proposer gets the upstream-2 summary files and\n    # skill mode \"default\" (no RunStore tools). --no-summary would instead\n    # withhold the summaries entirely, breaking parity with the organized arm.\n    arm_args=(--selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n  elif [ \"$arm\" = \"organized\" ]; then\n    arm_args=(--organized --selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n  else\n    printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n    return 0\n  fi\n\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s baseline=%s\\n[%s] LOG   %s\\n' \\\n    \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  # No --no-test-frontier: after the 20 iterations the optimizer automatically\n  # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n  # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n  setsid nohup python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --baseline-dir \"$BASELINE_DIR\" \\\n    --test-frontier-candidate-limit 1 \\\n    \"${arm_args[@]}\" \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n  echo \"error: baseline prime failed — see $status_file\" >&2\n  exit 1\n}\n\nfor arm in default organized; do\n  contains \"$ARMS\" \"$arm\" || continue\n  start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "7e699a8a3645dfdb1732855649a6105f26ccfdbe": {"t": 1785758239.7900686, "n": 12540, "h": "27e9eaf8f7997efe", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n#   * proposer = Codex CLI authenticated against Azure OpenAI (default), or the\n#                Claude Code CLI routed at a provider's anthropic-compatible\n#                endpoint. Pick with PROPOSER_AGENT=codex|claude — see the\n#                proposer knobs below and docs/TERMINUS_CODEX_AZURE.md.\n#   * solver   = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n#                SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n#                official DeepSeek API, not Together AI.\n#   * rollout  = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n#                bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n#   * default   arm: --selection-policy default\n#                    (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n#   * organized arm: --organized --selection-policy default\n#                    (upstream-2 summaries, skill mode \"organized-summaries\",\n#                     generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\n# DAYTONA_API_KEY is always required (every rollout is a Daytona sandbox). The\n# proposer's own secret check is PROPOSER_AGENT-dependent and runs after the\n# knobs block below.\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n  echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n  echo \"       a managed-cloud key from https://app.daytona.io.\" >&2\n  exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# ---- proposer (PROPOSER_AGENT=codex|claude) -----------------------------\nPROPOSER_AGENT=\"${PROPOSER_AGENT:-codex}\"\n\n# Codex proposer (Azure OpenAI) — used when PROPOSER_AGENT=codex. CODEX_MODEL\n# MUST be your Azure *deployment* name — it is forwarded as `codex exec -m`,\n# and config.toml's model_provider routes it to Azure. Keep it in sync with\n# `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\"   # empty => ~/.codex (must hold the Azure config.toml)\n\n# Claude Code proposer — used when PROPOSER_AGENT=claude. Azure OpenAI is NOT\n# anthropic-compatible, so the Claude proposer is routed at a provider's\n# anthropic-compatible endpoint instead. Configure your provider:\n#   CLAUDE_BASE_URL    the provider's anthropic-compatible base URL\n#   CLAUDE_MODEL       the model id that provider expects\n#   CLAUDE_API_KEY_ENV names the .env variable holding the provider API key\n#   CLAUDE_EFFORT      optional thinking effort (low|medium|high|xhigh|max)\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-}\"\nCLAUDE_API_KEY_ENV=\"${CLAUDE_API_KEY_ENV:-ANTHROPIC_AUTH_TOKEN}\"\nCLAUDE_EFFORT=\"${CLAUDE_EFFORT:-}\"\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to. SOLVER_MODEL is a litellm\n# <provider>/<model> id — set it to match how your DeepSeek-V4-Pro\n# provider/endpoint is configured.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n  echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n  echo \"         unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\n# ---- preflight: proposer ------------------------------------------------\n# Build the proposer CLI fragment and check the secret the chosen agent needs.\nproposer_args=()\nif [ \"$PROPOSER_AGENT\" = \"codex\" ]; then\n  if [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n    echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n    echo \"       against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n    exit 1\n  fi\n  proposer_args=(\n    --proposer-agent codex\n    --codex-model \"$CODEX_MODEL\"\n    --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n  )\n  [ -n \"$CODEX_HOME\" ] && proposer_args+=(--codex-home \"$CODEX_HOME\")\nelif [ \"$PROPOSER_AGENT\" = \"claude\" ]; then\n  claude_key=\"${!CLAUDE_API_KEY_ENV:-}\"\n  if [ -z \"$CLAUDE_BASE_URL\" ] || [ -z \"$CLAUDE_MODEL\" ]; then\n    echo \"error: PROPOSER_AGENT=claude needs CLAUDE_BASE_URL and CLAUDE_MODEL set\" >&2\n    echo \"       to your provider's anthropic-compatible endpoint and model id —\" >&2\n    echo \"       see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n    exit 1\n  fi\n  if [ -z \"$claude_key\" ]; then\n    echo \"error: \\$$CLAUDE_API_KEY_ENV is empty. PROPOSER_AGENT=claude reads the\" >&2\n    echo \"       provider API key from the .env variable named by CLAUDE_API_KEY_ENV.\" >&2\n    exit 1\n  fi\n  if ! command -v claude >/dev/null 2>&1; then\n    echo \"error: the 'claude' CLI is not on PATH. PROPOSER_AGENT=claude runs the\" >&2\n    echo \"       Claude Code CLI — install it: npm install -g @anthropic-ai/claude-code\" >&2\n    exit 1\n  fi\n  proposer_args=(\n    --proposer-agent claude\n    --claude-base-url \"$CLAUDE_BASE_URL\"\n    --claude-model \"$CLAUDE_MODEL\"\n    --claude-auth-token \"$claude_key\"\n  )\n  [ -n \"$CLAUDE_EFFORT\" ] && proposer_args+=(--claude-effort \"$CLAUDE_EFFORT\")\nelse\n  echo \"error: PROPOSER_AGENT must be 'codex' or 'claude'; got '$PROPOSER_AGENT'\" >&2\n  exit 1\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s proposer=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n  \"$(date -Is)\" \"$TS\" \"$PROPOSER_AGENT\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n  >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + the chosen proposer (proposer_args, built\n# in the preflight above). The per-run bits (--run-id, --iterations, arm flags,\n# test-frontier) are added by the callers.\ncommon_args=(\n  --terminus\n  --split \"$SPLIT\"\n  --limit \"$LIMIT\"\n  --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n  --terminus-env daytona\n  --terminus-solver-model \"$SOLVER_MODEL\"\n  --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n  --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n  --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n  --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n  --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n  --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n  \"${proposer_args[@]}\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n  if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n    return 0\n  fi\n  local prime_log=\"logs/${baseline_run_id}.log\"\n  printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n    \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n  python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$baseline_run_id\" \\\n    --iterations 0 \\\n    --selection-policy default \\\n    --no-test-frontier \\\n    > \"$prime_log\" 2>&1\n  local rc=$?\n  if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n      \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n    return 1\n  fi\n  printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n  local arm=\"$1\"\n  local arm_args=() run_id log_path\n  if [ \"$arm\" = \"default\" ]; then\n    # No summary flag: the proposer gets the upstream-2 summary files and\n    # skill mode \"default\" (no RunStore tools). --no-summary would instead\n    # withhold the summaries entirely, breaking parity with the organized arm.\n    arm_args=(--selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n  elif [ \"$arm\" = \"organized\" ]; then\n    arm_args=(--organized --selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n  else\n    printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n    return 0\n  fi\n\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s baseline=%s\\n[%s] LOG   %s\\n' \\\n    \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  # No --no-test-frontier: after the 20 iterations the optimizer automatically\n  # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n  # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n  setsid nohup python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --baseline-dir \"$BASELINE_DIR\" \\\n    --test-frontier-candidate-limit 1 \\\n    \"${arm_args[@]}\" \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n  echo \"error: baseline prime failed — see $status_file\" >&2\n  exit 1\n}\n\nfor arm in default organized; do\n  contains \"$ARMS\" \"$arm\" || continue\n  start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "75db5644133af8ced48316ce53738341568665c9": {"t": 1785758242.0079272, "n": 9907, "h": "acbc27cf71846bc9", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n#   * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n#                key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n#                there is no interactive Codex \"login\" for Azure.\n#   * solver   = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n#                SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n#                official DeepSeek API, not Together AI.\n#   * rollout  = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n#                bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n#   * default   arm: --selection-policy default\n#                    (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n#   * organized arm: --organized --selection-policy default\n#                    (upstream-2 summaries, skill mode \"organized-summaries\",\n#                     generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n  echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n  echo \"       a managed-cloud key from https://app.daytona.io.\" >&2\n  exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n  echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n  echo \"       against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n  exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\"   # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to. SOLVER_MODEL is a litellm\n# <provider>/<model> id — set it to match how your DeepSeek-V4-Pro\n# provider/endpoint is configured.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n  echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n  echo \"         unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n  \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n  >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n  --terminus\n  --split \"$SPLIT\"\n  --limit \"$LIMIT\"\n  --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n  --terminus-env daytona\n  --terminus-solver-model \"$SOLVER_MODEL\"\n  --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n  --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n  --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n  --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n  --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n  --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n  --proposer-agent codex\n  --codex-model \"$CODEX_MODEL\"\n  --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n  if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n    return 0\n  fi\n  local prime_log=\"logs/${baseline_run_id}.log\"\n  printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n    \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n  python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$baseline_run_id\" \\\n    --iterations 0 \\\n    --selection-policy default \\\n    --no-test-frontier \\\n    > \"$prime_log\" 2>&1\n  local rc=$?\n  if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n      \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n    return 1\n  fi\n  printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n  local arm=\"$1\"\n  local arm_args=() run_id log_path\n  if [ \"$arm\" = \"default\" ]; then\n    # No summary flag: the proposer gets the upstream-2 summary files and\n    # skill mode \"default\" (no RunStore tools). --no-summary would instead\n    # withhold the summaries entirely, breaking parity with the organized arm.\n    arm_args=(--selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n  elif [ \"$arm\" = \"organized\" ]; then\n    arm_args=(--organized --selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n  else\n    printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n    return 0\n  fi\n\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s baseline=%s\\n[%s] LOG   %s\\n' \\\n    \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  # No --no-test-frontier: after the 20 iterations the optimizer automatically\n  # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n  # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n  setsid nohup python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --baseline-dir \"$BASELINE_DIR\" \\\n    --test-frontier-candidate-limit 1 \\\n    \"${arm_args[@]}\" \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n  echo \"error: baseline prime failed — see $status_file\" >&2\n  exit 1\n}\n\nfor arm in default organized; do\n  contains \"$ARMS\" \"$arm\" || continue\n  start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "8e7c564f6382d5be813c7edb744b66ca05897968": {"t": 1785758243.9969687, "n": 9779, "h": "28dcee4d1c7cca94", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n#   * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n#                key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n#                there is no interactive Codex \"login\" for Azure.\n#   * solver   = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n#                SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n#                official DeepSeek API, not Together AI.\n#   * rollout  = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n#                bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n#   * default   arm: --selection-policy default\n#                    (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n#   * organized arm: --organized --selection-policy default\n#                    (upstream-2 summaries, skill mode \"organized-summaries\",\n#                     generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n  echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n  echo \"       a managed-cloud key from https://app.daytona.io.\" >&2\n  exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n  echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n  echo \"       against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n  exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\"   # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n  echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n  echo \"         unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n  \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n  >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n  --terminus\n  --split \"$SPLIT\"\n  --limit \"$LIMIT\"\n  --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n  --terminus-env daytona\n  --terminus-solver-model \"$SOLVER_MODEL\"\n  --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n  --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n  --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n  --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n  --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n  --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n  --proposer-agent codex\n  --codex-model \"$CODEX_MODEL\"\n  --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n  if [ -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n    return 0\n  fi\n  local prime_log=\"logs/${baseline_run_id}.log\"\n  printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n    \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n  python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$baseline_run_id\" \\\n    --iterations 0 \\\n    --selection-policy default \\\n    --no-test-frontier \\\n    > \"$prime_log\" 2>&1\n  local rc=$?\n  if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/optimizer_summary.json\" ]; then\n    printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n      \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n    return 1\n  fi\n  printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n  local arm=\"$1\"\n  local arm_args=() run_id log_path\n  if [ \"$arm\" = \"default\" ]; then\n    # No summary flag: the proposer gets the upstream-2 summary files and\n    # skill mode \"default\" (no RunStore tools). --no-summary would instead\n    # withhold the summaries entirely, breaking parity with the organized arm.\n    arm_args=(--selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n  elif [ \"$arm\" = \"organized\" ]; then\n    arm_args=(--organized --selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n  else\n    printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n    return 0\n  fi\n\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s baseline=%s\\n[%s] LOG   %s\\n' \\\n    \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  # No --no-test-frontier: after the 20 iterations the optimizer automatically\n  # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n  # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n  setsid nohup python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --baseline-dir \"$BASELINE_DIR\" \\\n    --test-frontier-candidate-limit 1 \\\n    \"${arm_args[@]}\" \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n  echo \"error: baseline prime failed — see $status_file\" >&2\n  exit 1\n}\n\nfor arm in default organized; do\n  contains \"$ARMS\" \"$arm\" || continue\n  start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "308805cd5f8184ff3e56367f978de84c6a1ac3c1": {"t": 1785758246.4679558, "n": 9767, "h": "7095b9d121d6b983", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n#   * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n#                key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n#                there is no interactive Codex \"login\" for Azure.\n#   * solver   = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n#                SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n#                official DeepSeek API, not Together AI.\n#   * rollout  = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n#                bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface. Both arms expose\n# the same upstream-2 summary files (evolution_summary.jsonl +\n# best_candidates.json), so the only variable across arms is the tools:\n#   * default   arm: --selection-policy default\n#                    (upstream-2 summaries, skill mode \"default\", no RunStore tools)\n#   * organized arm: --organized --selection-policy default\n#                    (upstream-2 summaries, skill mode \"organized-summaries\",\n#                     generates state.md and registers RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n  echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n  echo \"       a managed-cloud key from https://app.daytona.io.\" >&2\n  exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n  echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n  echo \"       against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n  exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\"   # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n  echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n  echo \"         unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n  \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n  >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n  --terminus\n  --split \"$SPLIT\"\n  --limit \"$LIMIT\"\n  --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n  --terminus-env daytona\n  --terminus-solver-model \"$SOLVER_MODEL\"\n  --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n  --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n  --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n  --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n  --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n  --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n  --proposer-agent codex\n  --codex-model \"$CODEX_MODEL\"\n  --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n  if [ -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n    printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n    return 0\n  fi\n  local prime_log=\"logs/${baseline_run_id}.log\"\n  printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n    \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n  python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$baseline_run_id\" \\\n    --iterations 0 \\\n    --selection-policy default \\\n    --no-test-frontier \\\n    > \"$prime_log\" 2>&1\n  local rc=$?\n  if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n    printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n      \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n    return 1\n  fi\n  printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n  local arm=\"$1\"\n  local arm_args=() run_id log_path\n  if [ \"$arm\" = \"default\" ]; then\n    # No summary flag: the proposer gets the upstream-2 summary files and\n    # skill mode \"default\" (no RunStore tools). --no-summary would instead\n    # withhold the summaries entirely, breaking parity with the organized arm.\n    arm_args=(--selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n  elif [ \"$arm\" = \"organized\" ]; then\n    arm_args=(--organized --selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n  else\n    printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n    return 0\n  fi\n\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s baseline=%s\\n[%s] LOG   %s\\n' \\\n    \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  # No --no-test-frontier: after the 20 iterations the optimizer automatically\n  # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n  # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n  setsid nohup python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --baseline-dir \"$BASELINE_DIR\" \\\n    --test-frontier-candidate-limit 1 \\\n    \"${arm_args[@]}\" \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n  echo \"error: baseline prime failed — see $status_file\" >&2\n  exit 1\n}\n\nfor arm in default organized; do\n  contains \"$ARMS\" \"$arm\" || continue\n  start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "c65db10e3af9ad72b5b2c5493eb6f2a877e8bfaf": {"t": 1785758247.7479098, "n": 9288, "h": "b3d0c7d8c6e1a569", "c": "#!/usr/bin/env bash\n# Terminal-Bench 2.0 (Terminus-KIRA) comparison experiment —\n# Codex proposer (Azure OpenAI) × DeepSeek-V4 solver, every Harbor trial in a\n# remote Daytona cloud sandbox.\n#\n# This is the experiment a teammate runs after pulling the repo. See\n# docs/TERMINUS_CODEX_AZURE.md for the full step-by-step setup.\n#\n#   * proposer = Codex CLI authenticated against Azure OpenAI. Auth is an API\n#                key in ~/.codex/config.toml (template: docs/codex_config.azure.toml);\n#                there is no interactive Codex \"login\" for Azure.\n#   * solver   = DeepSeek-V4 on YOUR OWN endpoint — override SOLVER_MODEL /\n#                SOLVER_BASE_URL / SOLVER_API_KEY_ENV. Defaults point at the\n#                official DeepSeek API, not Together AI.\n#   * rollout  = Daytona cloud sandbox (--terminus-env daytona). Concurrency is\n#                bounded by your Daytona account's concurrent-sandbox quota.\n#\n# Scale: 20 evolution iterations, 20 tasks/iter (first 20 of the 30-task hard\n# split), 2 attempts/task, all 40 trials per iteration parallel on Daytona.\n#\n# Two arms — identical except for the RunStore tool surface:\n#   * default   arm: --no-summary --selection-policy default\n#                    (no summary files, no RunStore tools)\n#   * organized arm: --organized  --selection-policy default\n#                    (state.md + RunStore tools)\n# Both arms share ONE primed iter-0 KIRA baseline (prime_terminus_baseline),\n# reused via --baseline-dir so the seed frontier is not paid for twice.\n#\n# After the 20 iterations each arm automatically evaluates its best train\n# Pareto-frontier candidate on the full TB2 test split (the 59 held-out tasks).\n#\n# Secrets are read from .env (git-ignored). Copy .env.example to .env and fill\n# it in — see docs/TERMINUS_CODEX_AZURE.md. NEVER commit the real .env.\n#\n# Each arm is detached with `setsid` so it survives the parent shell. The\n# baseline prime runs in the foreground (it must finish before the arms start),\n# so launch the whole script under `nohup`/`tmux` if your SSH session is flaky.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\nROOT=\"$(pwd)\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\n# ---- preflight: secrets -------------------------------------------------\nif [ -z \"${DAYTONA_API_KEY:-}\" ]; then\n  echo \"error: DAYTONA_API_KEY not set. Copy .env.example to .env and fill in\" >&2\n  echo \"       a managed-cloud key from https://app.daytona.io.\" >&2\n  exit 1\nfi\nif [ -z \"${AZURE_OPENAI_API_KEY:-}\" ]; then\n  echo \"error: AZURE_OPENAI_API_KEY not set. The Codex proposer authenticates\" >&2\n  echo \"       against Azure OpenAI via it — see docs/TERMINUS_CODEX_AZURE.md.\" >&2\n  exit 1\nfi\n\n# ---- knobs (env vars) ---------------------------------------------------\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nARMS=\"${ARMS:-default,organized}\"\nITERATIONS=\"${ITERATIONS:-20}\"\nSPLIT=\"${SPLIT:-train}\"\nLIMIT=\"${LIMIT:-20}\"\nROLLOUT_TRIALS=\"${ROLLOUT_TRIALS:-2}\"\nROLLOUT_CONCURRENCY=\"${ROLLOUT_CONCURRENCY:-40}\"\nAGENT_TIMEOUT_MULT=\"${AGENT_TIMEOUT_MULT:-2.0}\"\nENV_KWARGS=\"${ENV_KWARGS-auto_stop_interval_mins=20,auto_delete_interval_mins=120}\"\nJOB_TIMEOUT_S=\"${JOB_TIMEOUT_S:-21600}\"\nTERMINUS_SOURCE_PATH=\"${TERMINUS_SOURCE_PATH:-references/vendor/meta-harness/reference_examples/terminal_bench_2}\"\nTERMINUS_TEST_TASKS=\"${TERMINUS_TEST_TASKS:-}\"\nDRY_RUN=\"${DRY_RUN:-0}\"\n\n# Codex proposer (Azure OpenAI). CODEX_MODEL MUST be your Azure *deployment*\n# name — it is forwarded as `codex exec -m`, and config.toml's model_provider\n# routes it to Azure. Keep it in sync with `model` in ~/.codex/config.toml.\nCODEX_MODEL=\"${CODEX_MODEL:-gpt-5.1-codex}\"\nCODEX_REASONING_EFFORT=\"${CODEX_REASONING_EFFORT:-high}\"\nCODEX_HOME=\"${CODEX_HOME:-}\"   # empty => ~/.codex (must hold the Azure config.toml)\n\n# Terminus solver — DeepSeek-V4 on your own endpoint. Override all three to\n# point at the endpoint you have access to.\nSOLVER_MODEL=\"${SOLVER_MODEL:-openai/deepseek-v4-pro}\"\nSOLVER_BASE_URL=\"${SOLVER_BASE_URL:-https://api.deepseek.com/v1}\"\nSOLVER_API_KEY_ENV=\"${SOLVER_API_KEY_ENV:-DEEPSEEK_API_KEY}\"\nREASONING_EFFORT=\"${REASONING_EFFORT:-high}\"\n\nif [ -z \"${!SOLVER_API_KEY_ENV:-}\" ]; then\n  echo \"warning: ${SOLVER_API_KEY_ENV} not set; the DeepSeek-V4 solver will fail\" >&2\n  echo \"         unless DRY_RUN=1. Set it in .env or export it.\" >&2\nfi\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_terminus_codex_azure_daytona_${TS}.status\"\n: > \"$status_file\"\nprintf '[%s] LAUNCHER start ts=%s iter=%s arms=%s split=%s limit=%s concurrency=%s\\n' \\\n  \"$(date -Is)\" \"$TS\" \"$ITERATIONS\" \"$ARMS\" \"$SPLIT\" \"$LIMIT\" \"$ROLLOUT_CONCURRENCY\" \\\n  >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\n# ---- shared CLI fragment ------------------------------------------------\n# Everything common to the baseline prime and both arms: terminus dataset +\n# Daytona rollout + DeepSeek solver + Codex(Azure) proposer. The per-run bits\n# (--run-id, --iterations, arm flags, test-frontier) are added by the callers.\ncommon_args=(\n  --terminus\n  --split \"$SPLIT\"\n  --limit \"$LIMIT\"\n  --terminus-source-path \"$TERMINUS_SOURCE_PATH\"\n  --terminus-env daytona\n  --terminus-solver-model \"$SOLVER_MODEL\"\n  --terminus-solver-base-url \"$SOLVER_BASE_URL\"\n  --terminus-solver-api-key-env \"$SOLVER_API_KEY_ENV\"\n  --terminus-rollout-trials \"$ROLLOUT_TRIALS\"\n  --terminus-rollout-concurrency \"$ROLLOUT_CONCURRENCY\"\n  --terminus-agent-timeout-multiplier \"$AGENT_TIMEOUT_MULT\"\n  --terminus-job-timeout-s \"$JOB_TIMEOUT_S\"\n  --proposer-agent codex\n  --codex-model \"$CODEX_MODEL\"\n  --codex-reasoning-effort \"$CODEX_REASONING_EFFORT\"\n)\n[ -n \"$ENV_KWARGS\" ] && common_args+=(--terminus-env-kwargs \"$ENV_KWARGS\")\n[ -n \"$REASONING_EFFORT\" ] && common_args+=(--terminus-reasoning-effort \"$REASONING_EFFORT\")\n[ -n \"$TERMINUS_TEST_TASKS\" ] && common_args+=(--terminus-test-tasks \"$TERMINUS_TEST_TASKS\")\n[ -n \"$CODEX_HOME\" ] && common_args+=(--codex-home \"$CODEX_HOME\")\n[ \"$DRY_RUN\" = \"1\" ] && common_args+=(--dry-run)\n\n# ---- shared primed baseline --------------------------------------------\n# One iter-0 run produces the KIRA seed frontier; both arms reuse it via\n# --baseline-dir. The count check in the optimizer requires the prime to use\n# the SAME --split/--limit as the arms, which it does (common_args).\nbaseline_run_id=\"terminus_kira_codex_azure_daytona_baseline_${SPLIT}_limit${LIMIT}_${TS}\"\nBASELINE_DIR=\"${BASELINE_DIR:-runs/${baseline_run_id}}\"\n\nprime_terminus_baseline() {\n  if [ -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n    printf '[%s] BASELINE_REUSE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n    return 0\n  fi\n  local prime_log=\"logs/${baseline_run_id}.log\"\n  printf '[%s] BASELINE_PRIME -> %s\\n[%s] BASELINE_LOG %s\\n' \\\n    \"$(date -Is)\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$prime_log\" >> \"$status_file\"\n  python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$baseline_run_id\" \\\n    --iterations 0 \\\n    --selection-policy default \\\n    --no-test-frontier \\\n    > \"$prime_log\" 2>&1\n  local rc=$?\n  if [ \"$rc\" -ne 0 ] || [ ! -f \"${BASELINE_DIR}/run_summary.json\" ]; then\n    printf '[%s] BASELINE_PRIME_FAIL rc=%s log=%s\\n' \\\n      \"$(date -Is)\" \"$rc\" \"$prime_log\" >> \"$status_file\"\n    return 1\n  fi\n  printf '[%s] BASELINE_PRIME_DONE %s\\n' \"$(date -Is)\" \"$BASELINE_DIR\" >> \"$status_file\"\n}\n\n# ---- one optimization arm ----------------------------------------------\nstart_one() {\n  local arm=\"$1\"\n  local arm_args=() run_id log_path\n  if [ \"$arm\" = \"default\" ]; then\n    arm_args=(--no-summary --selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_default_${SPLIT}_${TS}\"\n  elif [ \"$arm\" = \"organized\" ]; then\n    arm_args=(--organized --selection-policy default)\n    run_id=\"terminus_kira_codex_azure_daytona_organized_${SPLIT}_${TS}\"\n  else\n    printf '[%s] SKIP unknown_arm=%s\\n' \"$(date -Is)\" \"$arm\" >> \"$status_file\"\n    return 0\n  fi\n\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s baseline=%s\\n[%s] LOG   %s\\n' \\\n    \"$(date -Is)\" \"$run_id\" \"$BASELINE_DIR\" \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  # No --no-test-frontier: after the 20 iterations the optimizer automatically\n  # evaluates the best train-frontier candidate (--test-frontier-candidate-limit 1)\n  # on the full TB2 test split (--test-frontier-limit 0 = all 59 held-out tasks).\n  setsid nohup python -m optimizer1.cli optimize \\\n    \"${common_args[@]}\" \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --baseline-dir \"$BASELINE_DIR\" \\\n    --test-frontier-candidate-limit 1 \\\n    \"${arm_args[@]}\" \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\n# Prime the shared baseline first (foreground), then launch both arms parallel.\nprime_terminus_baseline || {\n  echo \"error: baseline prime failed — see $status_file\" >&2\n  exit 1\n}\n\nfor arm in default organized; do\n  contains \"$ARMS\" \"$arm\" || continue\n  start_one \"$arm\"\ndone\n\nprintf '[%s] LAUNCHER done\\n' \"$(date -Is)\" >> \"$status_file\"\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}}, "scripts/launch_deepseek_v4_pro_pure_default_notrace.sh": {"8624ef6a03f3e1e6fff81c2dbbee15e39bf38383": {"t": 1785745960.4925592, "n": 3986, "h": "ab14dd6792c05f82", "c": "#!/usr/bin/env bash\n# Launch the pure default no-trace control runs across LoCoMo and\n# LongMemEval-S. This is the baseline arm used to compare against\n# organized default: default selection, clean patch base, no state.md,\n# and no proposer-facing trace/RunStore MCP tools.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\nif [ -z \"${DEEPSEEK_API_KEY:-}\" ]; then\n  printf 'fatal: DEEPSEEK_API_KEY is not set; populate .env or export it.\\n' >&2\n  exit 2\nfi\n\nunset ANTHROPIC_API_KEY\nexport ANTHROPIC_AUTH_TOKEN=\"${DEEPSEEK_API_KEY}\"\nexport ANTHROPIC_BASE_URL=\"${ANTHROPIC_BASE_URL:-https://api.deepseek.com/anthropic}\"\nexport ANTHROPIC_MODEL=\"${ANTHROPIC_MODEL:-deepseek-v4-pro[1m]}\"\n\nif [ -n \"${TOGETHER_API_KEY:-}\" ]; then\n  export OPENAI_API_KEY=\"${TOGETHER_API_KEY}\"\n  export OPENAI_BASE_URL=\"https://api.together.xyz/v1\"\nfi\nexport DIFF_EMBEDDING_MODEL=\"${DIFF_EMBEDDING_MODEL:-BAAI/bge-large-en-v1.5}\"\n\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nITERATIONS=\"${ITERATIONS:-50}\"\nTASKS=\"${TASKS:-locomo,longmemeval}\"\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-deepseek-v4-pro[1m]}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-https://api.deepseek.com/anthropic}\"\nEVAL_WORKERS=\"${EVAL_WORKERS:-128}\"\nEVAL_MODEL=\"${EVAL_MODEL:-/data/home/yuhan/model_zoo/Qwen3-8B}\"\nEVAL_BASE_URL=\"${EVAL_BASE_URL:-http://127.0.0.1:8002/v1}\"\nDOCKER_USER_SPEC=\"${DOCKER_USER_SPEC:-$(id -u):$(id -g)}\"\nDOCKER_IMAGE=\"${DOCKER_IMAGE:-docker-claude:latest}\"\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_deepseek_v4_pro_pure_default_notrace_${TS}.status\"\n: > \"$status_file\"\nprintf 'timestamp=%s\\n' \"$TS\" >> \"$status_file\"\nprintf 'mode=pure_default_no_trace_harness\\n' >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\nstart_one() {\n  local task=\"$1\"\n  local task_label task_args=() run_id log_path\n\n  if [ \"$task\" = \"locomo\" ]; then\n    task_label=\"locomo\"\n    task_args=(--locomo)\n  elif [ \"$task\" = \"longmemeval\" ]; then\n    task_label=\"longmemeval_s\"\n    task_args=(--longmemeval --longmemeval-variant s)\n  else\n    printf '[%s] SKIP unknown_task=%s\\n' \"$(date -Is)\" \"$task\" >> \"$status_file\"\n    return 0\n  fi\n\n  run_id=\"${task_label}_deepseek_v4_pro_pure_default_notrace_iter${ITERATIONS}_${TS}\"\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n  printf '[%s] LOG   %s\\n' \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  setsid python -m optimizer1.cli optimize \\\n    \"${task_args[@]}\" \\\n    --selection-policy default \\\n    --proposer-no-trace-harness-section \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --split train \\\n    --scaffolds memgpt_source \\\n    --scaffold-extra-json @configs/source_memory.example.json \\\n    --eval-workers \"$EVAL_WORKERS\" \\\n    --model \"$EVAL_MODEL\" \\\n    --base-url \"$EVAL_BASE_URL\" \\\n    --proposer-agent claude \\\n    --claude-model \"$CLAUDE_MODEL\" \\\n    --claude-base-url \"$CLAUDE_BASE_URL\" \\\n    --claude-effort max \\\n    --force-budget high \\\n    --proposer-sandbox docker \\\n    --proposer-docker-image \"$DOCKER_IMAGE\" \\\n    --proposer-docker-user \"$DOCKER_USER_SPEC\" \\\n    --proposer-docker-home /tmp \\\n    --proposer-docker-env DEEPSEEK_API_KEY \\\n    --proposer-docker-env ANTHROPIC_AUTH_TOKEN \\\n    --proposer-docker-env ANTHROPIC_BASE_URL \\\n    --proposer-docker-env ANTHROPIC_MODEL \\\n    --proposer-docker-env OPENAI_API_KEY \\\n    --proposer-docker-env OPENAI_BASE_URL \\\n    --proposer-docker-env DIFF_EMBEDDING_MODEL \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\nfor task in locomo longmemeval; do\n  contains \"$TASKS\" \"$task\" || continue\n  start_one \"$task\"\ndone\n\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}, "5cefec0bc430e944c304d1ae937b4defd0578a1f": {"t": 1785758237.4609592, "n": 3986, "h": "ab14dd6792c05f82", "c": "#!/usr/bin/env bash\n# Launch the pure default no-trace control runs across LoCoMo and\n# LongMemEval-S. This is the baseline arm used to compare against\n# organized default: default selection, clean patch base, no state.md,\n# and no proposer-facing trace/RunStore MCP tools.\nset -u -o pipefail\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")/..\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\nif [ -z \"${DEEPSEEK_API_KEY:-}\" ]; then\n  printf 'fatal: DEEPSEEK_API_KEY is not set; populate .env or export it.\\n' >&2\n  exit 2\nfi\n\nunset ANTHROPIC_API_KEY\nexport ANTHROPIC_AUTH_TOKEN=\"${DEEPSEEK_API_KEY}\"\nexport ANTHROPIC_BASE_URL=\"${ANTHROPIC_BASE_URL:-https://api.deepseek.com/anthropic}\"\nexport ANTHROPIC_MODEL=\"${ANTHROPIC_MODEL:-deepseek-v4-pro[1m]}\"\n\nif [ -n \"${TOGETHER_API_KEY:-}\" ]; then\n  export OPENAI_API_KEY=\"${TOGETHER_API_KEY}\"\n  export OPENAI_BASE_URL=\"https://api.together.xyz/v1\"\nfi\nexport DIFF_EMBEDDING_MODEL=\"${DIFF_EMBEDDING_MODEL:-BAAI/bge-large-en-v1.5}\"\n\nTS=\"${TS:-$(date +%Y%m%d_%H%M%S)}\"\nITERATIONS=\"${ITERATIONS:-50}\"\nTASKS=\"${TASKS:-locomo,longmemeval}\"\nCLAUDE_MODEL=\"${CLAUDE_MODEL:-deepseek-v4-pro[1m]}\"\nCLAUDE_BASE_URL=\"${CLAUDE_BASE_URL:-https://api.deepseek.com/anthropic}\"\nEVAL_WORKERS=\"${EVAL_WORKERS:-128}\"\nEVAL_MODEL=\"${EVAL_MODEL:-/data/home/yuhan/model_zoo/Qwen3-8B}\"\nEVAL_BASE_URL=\"${EVAL_BASE_URL:-http://127.0.0.1:8002/v1}\"\nDOCKER_USER_SPEC=\"${DOCKER_USER_SPEC:-$(id -u):$(id -g)}\"\nDOCKER_IMAGE=\"${DOCKER_IMAGE:-docker-claude:latest}\"\n\nmkdir -p logs runs\nstatus_file=\"logs/launch_deepseek_v4_pro_pure_default_notrace_${TS}.status\"\n: > \"$status_file\"\nprintf 'timestamp=%s\\n' \"$TS\" >> \"$status_file\"\nprintf 'mode=pure_default_no_trace_harness\\n' >> \"$status_file\"\n\ncontains() { case \",$1,\" in *\",$2,\"*) return 0;; *) return 1;; esac; }\n\nstart_one() {\n  local task=\"$1\"\n  local task_label task_args=() run_id log_path\n\n  if [ \"$task\" = \"locomo\" ]; then\n    task_label=\"locomo\"\n    task_args=(--locomo)\n  elif [ \"$task\" = \"longmemeval\" ]; then\n    task_label=\"longmemeval_s\"\n    task_args=(--longmemeval --longmemeval-variant s)\n  else\n    printf '[%s] SKIP unknown_task=%s\\n' \"$(date -Is)\" \"$task\" >> \"$status_file\"\n    return 0\n  fi\n\n  run_id=\"${task_label}_deepseek_v4_pro_pure_default_notrace_iter${ITERATIONS}_${TS}\"\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n    return 0\n  fi\n\n  log_path=\"logs/${run_id}.log\"\n  printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\" >> \"$status_file\"\n  printf '[%s] LOG   %s\\n' \"$(date -Is)\" \"$log_path\" >> \"$status_file\"\n\n  setsid python -m optimizer1.cli optimize \\\n    \"${task_args[@]}\" \\\n    --selection-policy default \\\n    --proposer-no-trace-harness-section \\\n    --run-id \"$run_id\" \\\n    --iterations \"$ITERATIONS\" \\\n    --split train \\\n    --scaffolds memgpt_source \\\n    --scaffold-extra-json @configs/source_memory.example.json \\\n    --eval-workers \"$EVAL_WORKERS\" \\\n    --model \"$EVAL_MODEL\" \\\n    --base-url \"$EVAL_BASE_URL\" \\\n    --proposer-agent claude \\\n    --claude-model \"$CLAUDE_MODEL\" \\\n    --claude-base-url \"$CLAUDE_BASE_URL\" \\\n    --claude-effort max \\\n    --force-budget high \\\n    --proposer-sandbox docker \\\n    --proposer-docker-image \"$DOCKER_IMAGE\" \\\n    --proposer-docker-user \"$DOCKER_USER_SPEC\" \\\n    --proposer-docker-home /tmp \\\n    --proposer-docker-env DEEPSEEK_API_KEY \\\n    --proposer-docker-env ANTHROPIC_AUTH_TOKEN \\\n    --proposer-docker-env ANTHROPIC_BASE_URL \\\n    --proposer-docker-env ANTHROPIC_MODEL \\\n    --proposer-docker-env OPENAI_API_KEY \\\n    --proposer-docker-env OPENAI_BASE_URL \\\n    --proposer-docker-env DIFF_EMBEDDING_MODEL \\\n    > \"$log_path\" 2>&1 < /dev/null &\n\n  local pid=$!\n  printf '[%s] PID   %s %s\\n' \"$(date -Is)\" \"$run_id\" \"$pid\" >> \"$status_file\"\n  printf '%s %s %s\\n' \"$pid\" \"$run_id\" \"$log_path\"\n}\n\nfor task in locomo longmemeval; do\n  contains \"$TASKS\" \"$task\" || continue\n  start_one \"$task\"\ndone\n\nprintf '\\nstatus: %s\\n' \"$status_file\"\n"}}, "scripts/launch_claudekimi_swebench_random_recent.sh": {"8624ef6a03f3e1e6fff81c2dbbee15e39bf38383": {"t": 1785745962.573957, "n": 3177, "h": "42755dde9110b3b3", "c": "#!/usr/bin/env bash\nset -u -o pipefail\n\ncd \"$(dirname \"$0\")/..\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\ntimestamp=\"${1:-$(date +%Y%m%d_%H%M%S)}\"\nmax_parallel=\"${MAX_PARALLEL:-3}\"\ndriver_status=\"logs/claudekimi_swebench_random_recent_${timestamp}.status\"\nmkdir -p logs runs\n: > \"$driver_status\"\n\nrun_job() {\n  local run_id=\"$1\"\n  shift\n  local log_path=\"logs/${run_id}.log\"\n  {\n    printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\"\n    printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\"\n  } >> \"$driver_status\"\n  \"$@\" > \"$log_path\" 2>&1\n  local status=$?\n  printf '[%s] END %s status=%s\\n' \"$(date -Is)\" \"$run_id\" \"$status\" >> \"$driver_status\"\n  return \"$status\"\n}\n\nwait_for_slot() {\n  local pattern=\"python -m optimizer1.cli optimize .*claudekimi_.*(3_fileaccess|bandit_v4_autobudget).*_${timestamp}\"\n  while [ \"$(pgrep -af \"$pattern\" | wc -l)\" -ge \"$max_parallel\" ]; do\n    sleep 30\n  done\n}\n\nstart_job() {\n  local run_id=\"$1\"\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir=runs/%s\\n' \"$(date -Is)\" \"$run_id\" \"$run_id\" >> \"$driver_status\"\n    return 0\n  fi\n  wait_for_slot\n  run_job \"$@\" &\n  pids+=(\"$!\")\n}\n\npids=()\n\nminiswe_run_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py run --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir} --model openai/deepseek-v4-flash --base-url https://api.deepseek.com/v1 --max-tokens 4096 --api-key-env DEEPSEEK_API_KEY\"\nminiswe_eval_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py eval --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir}\"\n\nminiswe_common=(\n  python -m optimizer1.cli optimize\n  --swebench\n  --iterations 20\n  --split train\n  --limit 30\n  --swebench-data-path data/swebench_verified_full.json\n  --eval-timeout-s 900\n  --eval-workers 10\n  --proposer-agent kimi\n  --proposer-sandbox docker\n  --proposer-docker-image docker-claude-kimi:latest\n  --proposer-docker-env KIMI_API_KEY\n  --proposer-docker-user 1023:1023\n  --proposer-docker-home /tmp\n  --mini-swe-agent-command \"$miniswe_run_command\"\n  --mini-swe-agent-eval-command \"$miniswe_eval_command\"\n)\n\nfor policy in random recent; do\n  run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_${policy}3_fileaccess_docker_iter20_trainfirst30_w10_t900_${timestamp}\"\n  start_job \"$run_id\" \"${miniswe_common[@]}\" \\\n    --run-id \"$run_id\" \\\n    --selection-policy \"$policy\"\ndone\n\n# bandit_v4 baseline on the same swebench config (current branch is bandit-v4,\n# so --selection-policy bandit dispatches to the v4 ref-iter selection rule).\nbandit_v4_run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_bandit_v4_autobudget_docker_iter20_trainfirst30_w10_t900_bw16_${timestamp}\"\nstart_job \"$bandit_v4_run_id\" \"${miniswe_common[@]}\" \\\n  --run-id \"$bandit_v4_run_id\" \\\n  --selection-policy bandit \\\n  --bandit-reward-window 16\n\nfailures=0\nfor pid in \"${pids[@]}\"; do\n  if ! wait \"$pid\"; then\n    failures=$((failures + 1))\n  fi\ndone\n\nprintf '[%s] COMPLETE failures=%s\\n' \"$(date -Is)\" \"$failures\" >> \"$driver_status\"\nexit \"$failures\"\n"}, "4466d9fe8d3d818e8bfb0759e520ccbbfa3c4296": {"t": 1785758238.607434, "n": 3177, "h": "42755dde9110b3b3", "c": "#!/usr/bin/env bash\nset -u -o pipefail\n\ncd \"$(dirname \"$0\")/..\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\ntimestamp=\"${1:-$(date +%Y%m%d_%H%M%S)}\"\nmax_parallel=\"${MAX_PARALLEL:-3}\"\ndriver_status=\"logs/claudekimi_swebench_random_recent_${timestamp}.status\"\nmkdir -p logs runs\n: > \"$driver_status\"\n\nrun_job() {\n  local run_id=\"$1\"\n  shift\n  local log_path=\"logs/${run_id}.log\"\n  {\n    printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\"\n    printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\"\n  } >> \"$driver_status\"\n  \"$@\" > \"$log_path\" 2>&1\n  local status=$?\n  printf '[%s] END %s status=%s\\n' \"$(date -Is)\" \"$run_id\" \"$status\" >> \"$driver_status\"\n  return \"$status\"\n}\n\nwait_for_slot() {\n  local pattern=\"python -m optimizer1.cli optimize .*claudekimi_.*(3_fileaccess|bandit_v4_autobudget).*_${timestamp}\"\n  while [ \"$(pgrep -af \"$pattern\" | wc -l)\" -ge \"$max_parallel\" ]; do\n    sleep 30\n  done\n}\n\nstart_job() {\n  local run_id=\"$1\"\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir=runs/%s\\n' \"$(date -Is)\" \"$run_id\" \"$run_id\" >> \"$driver_status\"\n    return 0\n  fi\n  wait_for_slot\n  run_job \"$@\" &\n  pids+=(\"$!\")\n}\n\npids=()\n\nminiswe_run_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py run --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir} --model openai/deepseek-v4-flash --base-url https://api.deepseek.com/v1 --max-tokens 4096 --api-key-env DEEPSEEK_API_KEY\"\nminiswe_eval_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py eval --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir}\"\n\nminiswe_common=(\n  python -m optimizer1.cli optimize\n  --swebench\n  --iterations 20\n  --split train\n  --limit 30\n  --swebench-data-path data/swebench_verified_full.json\n  --eval-timeout-s 900\n  --eval-workers 10\n  --proposer-agent kimi\n  --proposer-sandbox docker\n  --proposer-docker-image docker-claude-kimi:latest\n  --proposer-docker-env KIMI_API_KEY\n  --proposer-docker-user 1023:1023\n  --proposer-docker-home /tmp\n  --mini-swe-agent-command \"$miniswe_run_command\"\n  --mini-swe-agent-eval-command \"$miniswe_eval_command\"\n)\n\nfor policy in random recent; do\n  run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_${policy}3_fileaccess_docker_iter20_trainfirst30_w10_t900_${timestamp}\"\n  start_job \"$run_id\" \"${miniswe_common[@]}\" \\\n    --run-id \"$run_id\" \\\n    --selection-policy \"$policy\"\ndone\n\n# bandit_v4 baseline on the same swebench config (current branch is bandit-v4,\n# so --selection-policy bandit dispatches to the v4 ref-iter selection rule).\nbandit_v4_run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_bandit_v4_autobudget_docker_iter20_trainfirst30_w10_t900_bw16_${timestamp}\"\nstart_job \"$bandit_v4_run_id\" \"${miniswe_common[@]}\" \\\n  --run-id \"$bandit_v4_run_id\" \\\n  --selection-policy bandit \\\n  --bandit-reward-window 16\n\nfailures=0\nfor pid in \"${pids[@]}\"; do\n  if ! wait \"$pid\"; then\n    failures=$((failures + 1))\n  fi\ndone\n\nprintf '[%s] COMPLETE failures=%s\\n' \"$(date -Is)\" \"$failures\" >> \"$driver_status\"\nexit \"$failures\"\n"}, "2ef73500ed812f64de46f0d6c34e4993a875aef3": {"t": 1785758240.5602186, "n": 3173, "h": "b11760d12df08dfa", "c": "#!/usr/bin/env bash\nset -u -o pipefail\n\ncd \"$(dirname \"$0\")/..\"\n\nif [ -f .env ]; then\n  set -a\n  # shellcheck disable=SC1091\n  source .env\n  set +a\nfi\n\ntimestamp=\"${1:-$(date +%Y%m%d_%H%M%S)}\"\nmax_parallel=\"${MAX_PARALLEL:-3}\"\ndriver_status=\"logs/claudekimi_swebench_random_recent_${timestamp}.status\"\nmkdir -p logs runs\n: > \"$driver_status\"\n\nrun_job() {\n  local run_id=\"$1\"\n  shift\n  local log_path=\"logs/${run_id}.log\"\n  {\n    printf '[%s] START %s\\n' \"$(date -Is)\" \"$run_id\"\n    printf '[%s] LOG %s\\n' \"$(date -Is)\" \"$log_path\"\n  } >> \"$driver_status\"\n  \"$@\" > \"$log_path\" 2>&1\n  local status=$?\n  printf '[%s] END %s status=%s\\n' \"$(date -Is)\" \"$run_id\" \"$status\" >> \"$driver_status\"\n  return \"$status\"\n}\n\nwait_for_slot() {\n  local pattern=\"python -m memomemo.cli optimize .*claudekimi_.*(3_fileaccess|bandit_v4_autobudget).*_${timestamp}\"\n  while [ \"$(pgrep -af \"$pattern\" | wc -l)\" -ge \"$max_parallel\" ]; do\n    sleep 30\n  done\n}\n\nstart_job() {\n  local run_id=\"$1\"\n  if [ -d \"runs/${run_id}\" ]; then\n    printf '[%s] SKIP %s existing_run_dir=runs/%s\\n' \"$(date -Is)\" \"$run_id\" \"$run_id\" >> \"$driver_status\"\n    return 0\n  fi\n  wait_for_slot\n  run_job \"$@\" &\n  pids+=(\"$!\")\n}\n\npids=()\n\nminiswe_run_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py run --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir} --model openai/deepseek-v4-flash --base-url https://api.deepseek.com/v1 --max-tokens 4096 --api-key-env DEEPSEEK_API_KEY\"\nminiswe_eval_command=\"python /data/home/yuhan/MemoMemo/scripts/run_miniswe_swebench_single.py eval --source-path {source_path} --instance-path {instance_path} --patch-path {patch_path} --task-dir {task_dir}\"\n\nminiswe_common=(\n  python -m memomemo.cli optimize\n  --swebench\n  --iterations 20\n  --split train\n  --limit 30\n  --swebench-data-path data/swebench_verified_full.json\n  --eval-timeout-s 900\n  --eval-workers 10\n  --proposer-agent kimi\n  --proposer-sandbox docker\n  --proposer-docker-image docker-claude-kimi:latest\n  --proposer-docker-env KIMI_API_KEY\n  --proposer-docker-user 1023:1023\n  --proposer-docker-home /tmp\n  --mini-swe-agent-command \"$miniswe_run_command\"\n  --mini-swe-agent-eval-command \"$miniswe_eval_command\"\n)\n\nfor policy in random recent; do\n  run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_${policy}3_fileaccess_docker_iter20_trainfirst30_w10_t900_${timestamp}\"\n  start_job \"$run_id\" \"${miniswe_common[@]}\" \\\n    --run-id \"$run_id\" \\\n    --selection-policy \"$policy\"\ndone\n\n# bandit_v4 baseline on the same swebench config (current branch is bandit-v4,\n# so --selection-policy bandit dispatches to the v4 ref-iter selection rule).\nbandit_v4_run_id=\"swebench_miniswe_deepseek_v4_flash_claudekimi_bandit_v4_autobudget_docker_iter20_trainfirst30_w10_t900_bw16_${timestamp}\"\nstart_job \"$bandit_v4_run_id\" \"${miniswe_common[@]}\" \\\n  --run-id \"$bandit_v4_run_id\" \\\n  --selection-policy bandit \\\n  --bandit-reward-window 16\n\nfailures=0\nfor pid in \"${pids[@]}\"; do\n  if ! wait \"$pid\"; then\n    failures=$((failures + 1))\n  fi\ndone\n\nprintf '[%s] COMPLETE failures=%s\\n' \"$(date -Is)\" \"$failures\" >> \"$driver_status\"\nexit \"$failures\"\n"}}, "references/vendor/MemGPT/.github/workflows/reusable-test-workflow.yml": {"8624ef6a03f3e1e6fff81c2dbbee15e39bf38383": {"t": 1786068224.2888663, "n": 19451, "h": "67ecde71d5879922", "c": "name: Reusable Test Workflow\n\non:\n  workflow_call:\n    inputs:\n      test-type:\n        description: 'Type of tests to run (unit, integration, docker, send-message, sqlite)'\n        required: true\n        type: string\n      core-directory:\n        description: 'Working directory for commands. Uses . (root) by default.'\n        required: false\n        type: string\n        default: '.'\n      install-args:\n        description: 'uv sync arguments'\n        required: true\n        type: string\n      test-command:\n        description: 'Command to run tests'\n        required: false\n        type: string\n        default: 'uv run --frozen pytest -svv'\n      test-path-prefix:\n        description: 'Prefix for test path (e.g., tests/)'\n        required: false\n        type: string\n        default: 'tests/'\n      timeout-minutes:\n        description: 'Timeout in minutes'\n        required: false\n        type: number\n        default: 15\n      runner:\n        description: 'Runner to use'\n        required: false\n        type: string\n        default: '[\"self-hosted\", \"small\"]'\n      matrix-strategy:\n        description: 'JSON string for matrix strategy'\n        required: false\n        type: string\n        default: '{}'\n      changed-files-pattern:\n        description: 'Pattern for changed files detection'\n        required: false\n        type: string\n        default: |\n          **\n          .github/workflows/reusable-test-workflow.yml\n      skip-fern-generation:\n        description: 'Skip Fern SDK generation'\n        required: false\n        type: boolean\n        default: false\n      use-docker:\n        description: 'Use Docker for tests'\n        required: false\n        type: boolean\n        default: false\n      ref:\n        description: 'Git ref to wait for checks on'\n        required: false\n        type: string\n        default: ${{ github.sha }}\n      use-redis:\n        description: 'Use Redis for tests'\n        required: false\n        type: boolean\n        default: false\n      is-external-pr:\n        description: 'Whether this is an external PR that needs protection'\n        required: false\n        type: boolean\n        default: false\n\njobs:\n  changed-files:\n    runs-on: ${{ fromJSON(inputs.runner) }}\n    name: changed-files\n    outputs:\n      all_changed_files: ${{ steps.changed-files.outputs.all_changed_files }}\n      any_changed: ${{ steps.changed-files.outputs.any_changed }}\n    steps:\n      - uses: actions/checkout@v4\n        with:\n          repository: ${{ github.event.pull_request.head.repo.full_name }}\n          ref: ${{ github.event.pull_request.head.ref }}\n          fetch-depth: 0\n      - name: Get changed files\n        id: changed-files\n        uses: tj-actions/changed-files@v46.0.4\n        with:\n          files: ${{ inputs.changed-files-pattern }}\n\n  cache-check:\n    needs: [changed-files]\n    runs-on: ${{ fromJSON(inputs.runner) }}\n    name: Check cache key\n    outputs:\n      cache_key: ${{ steps.cache-key.outputs.key }}\n      cache_hit: ${{ steps.cache.outputs.cache-hit }}\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v4\n        with:\n          repository: ${{ github.event.pull_request.head.repo.full_name }}\n          ref: ${{ github.event.pull_request.head.ref }}\n\n      - name: Generate cache key\n        if: inputs.skip-fern-generation != true || (!contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi.json') && !contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi-overrides.yml'))\n        id: cache-key\n        run: |\n          echo \"key=sdk-${{ github.ref_name }}-${{ hashFiles('fern/*', 'pyproject.toml') }}\" >> $GITHUB_OUTPUT\n\n      - name: Restore SDK cache\n        # skip if \"skip-fern-generation\" is true or if the upstream workflow would've generated an sdk preview (changes to openapi files)\n        if: inputs.skip-fern-generation != true || (!contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi.json') && !contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi-overrides.yml'))\n        id: cache\n        uses: actions/cache/restore@v4\n        with:\n          path: |\n            fern/.preview/fern-python-sdk/\n          key: ${{ steps.cache-key.outputs.key }}\n          fail-on-cache-miss: false\n\n  block-until-sdk-preview-finishes:\n    needs: [changed-files, cache-check]\n    if: |\n      needs.cache-check.outputs.cache_hit != 'true'\n    timeout-minutes: ${{ inputs.timeout-minutes }}\n    runs-on: ${{ fromJSON(inputs.runner) }}\n    name: block-until-sdk-preview-finishes\n    steps:\n      - name: Debug ref information\n        run: |\n          echo \"Input ref: ${{ inputs.ref }}\"\n          echo \"GitHub SHA: ${{ github.sha }}\"\n          echo \"GitHub ref: ${{ github.ref }}\"\n          echo \"PR head SHA: ${{ github.event.pull_request.head.sha }}\"\n          echo \"Event name: ${{ github.event_name }}\"\n\n      - name: Wait for Preview SDK workflow\n        if: inputs.skip-fern-generation != true || (!contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi.json') && !contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi-overrides.yml'))\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          echo \"Waiting for 'preview-python-sdk' check to complete on ref: ${{ inputs.ref }}\"\n\n          # Wait for the check to complete with timeout\n          timeout_seconds=1800\n          interval_seconds=60\n          elapsed=0\n\n          while [ $elapsed -lt $timeout_seconds ]; do\n            echo \"Checking status... (elapsed: ${elapsed}s)\"\n\n            # Get check runs using pr checks syntax with branch name or PR number\n            if [ \"${{ github.event_name }}\" = \"pull_request\" ]; then\n              pr_identifier=\"${{ github.event.pull_request.number }}\"\n            else\n              pr_identifier=\"${{ github.ref_name }}\"\n            fi\n\n            check_info=$(gh pr checks \"$pr_identifier\" -R ${{ github.repository }} --json name,state,startedAt \\\n              | jq -r '.[] | select(.name == \"preview-python-sdk\") | [.startedAt, .state] | @tsv' | sort -r | head -1 | cut -f2)\n\n            if [ -n \"$check_info\" ]; then\n              echo \"Check state: $check_info\"\n\n              if [ \"$check_info\" = \"SUCCESS\" ] || [ \"$check_info\" = \"SKIPPED\" ]; then\n                echo \"Check completed with state: $check_info\"\n                exit 0\n              elif [ \"$check_info\" = \"FAILURE\" ] || [ \"$check_info\" = \"CANCELLED\" ]; then\n                echo \"❌ Preview Python SDK build failed with state: $check_info\"\n                echo \"🚫 Blocking dependent test jobs to prevent extraneous failures\"\n                echo \"📋 To fix: Check the 'preview-python-sdk' job logs for build errors\"\n                exit 1\n              fi\n            else\n              echo \"Check 'preview-python-sdk' not found yet\"\n            fi\n\n            sleep $interval_seconds\n            elapsed=$((elapsed + interval_seconds))\n          done\n\n          echo \"Timeout waiting for check to complete\"\n          exit 1\n\n  test-run:\n    needs: [changed-files, block-until-sdk-preview-finishes]\n    if: |\n      always() &&\n      needs.changed-files.outputs.any_changed == 'true' &&\n      (needs.block-until-sdk-preview-finishes.result == 'success' ||\n       needs.block-until-sdk-preview-finishes.result == 'skipped')\n\n    runs-on: ${{ fromJSON(inputs.runner) }}\n    timeout-minutes: ${{ inputs.timeout-minutes }}\n    strategy: ${{ fromJSON(inputs.matrix-strategy) }}\n\n    services:\n      postgres:\n        image: pgvector/pgvector:pg17\n        ports:\n          # avoids conflict with docker postgres\n          - ${{ inputs.use-docker && '9999:5432' || '5432:5432' }}\n        env:\n          POSTGRES_HOST_AUTH_METHOD: trust\n          POSTGRES_DB: postgres\n          POSTGRES_USER: postgres\n        options: >-\n          --health-cmd pg_isready\n          --health-interval 10s\n          --health-timeout 5s\n          --health-retries 5\n      redis:\n        image: ${{ inputs.use-redis && 'redis:8-alpine' || '' }}\n        options: >-\n          --health-cmd \"redis-cli ping\"\n          --health-interval 10s\n          --health-timeout 5s\n          --health-retries 5\n        ports:\n          - 6379:6379\n\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v4\n        with:\n          repository: ${{ github.event.pull_request.head.repo.full_name }}\n          ref: ${{ github.event.pull_request.head.ref }}\n\n      - name: Install uv\n        run: |\n          curl -LsSf https://astral.sh/uv/install.sh | sh\n          echo \"$HOME/.cargo/bin\" >> $GITHUB_PATH\n\n      - name: Set core directory\n        id: detect-core-dir\n        run: |\n          echo \"dir=${{ inputs.core-directory }}\" >> $GITHUB_OUTPUT\n          echo \"detected=manual\" >> $GITHUB_OUTPUT\n          echo \"Using core directory: $(cat $GITHUB_OUTPUT | grep '^dir=' | cut -d'=' -f2)\"\n\n      - name: Generate cache key\n        if: inputs.skip-fern-generation != true || (!contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi.json') && !contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi-overrides.yml'))\n        id: cache-key\n        run: |\n          echo \"key=sdk-${{ github.ref_name }}-${{ hashFiles('fern/*', 'pyproject.toml') }}\" >> $GITHUB_OUTPUT\n\n      - name: Restore SDK cache\n        # skip if \"skip-fern-generation\" is true or if the upstream workflow would've generated an sdk preview (changes to openapi files)\n        if: inputs.skip-fern-generation != true || (!contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi.json') && !contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi-overrides.yml'))\n        id: restore-sdk-cache\n        uses: actions/cache/restore@v4\n        with:\n          path: |\n            fern/.preview/fern-python-sdk/\n          key: ${{ steps.cache-key.outputs.key }}\n          fail-on-cache-miss: false\n\n      - name: Check SDK cache availability\n        if: (inputs.skip-fern-generation != true || (!contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi.json') && !contains(needs.changed-files.outputs.all_changed_files, 'fern/openapi-overrides.yml'))) && steps.restore-sdk-cache.outputs.cache-hit != 'true'\n        run: |\n          echo \"❌ Preview Python SDK cache expired or missing!\"\n          echo \"📦 Cache key: ${{ steps.cache-key.outputs.key }}\"\n          echo \"🔄 To fix: Re-run the 'preview-python-sdk' workflow job to regenerate the SDK\"\n          echo \"💡 This can happen when:\"\n          echo \"   - The cache entry has expired\"\n          echo \"   - Dependencies in fern/* or pyproject.toml have changed\"\n          echo \"   - The preview-python-sdk job hasn't run successfully for this branch/commit\"\n          exit 1\n\n      - name: Install dependencies with retry\n        shell: bash\n        working-directory: .\n        run: |\n          uv sync --no-install-project ${{ inputs.install-args }}\n\n      - name: Install custom SDK\n        if: inputs.skip-fern-generation != true\n        working-directory: .\n        run: |\n          echo \"Fixing Fern SDK pyproject.toml for uv compatibility...\"\n          SDK_PYPROJECT=\"fern/.preview/fern-python-sdk/pyproject.toml\"\n          VERSION=$(grep -A 10 '^\\[tool\\.poetry\\]' \"$SDK_PYPROJECT\" | grep '^version' | head -1 | cut -d'\"' -f2)\n          head -n 2 < fern/.preview/fern-python-sdk/pyproject.toml > fern/.preview/fern-python-sdk/pyproject.toml.tmp\n          echo \"version = \\\"$VERSION\\\"\" >> fern/.preview/fern-python-sdk/pyproject.toml.tmp\n          tail -n +3 fern/.preview/fern-python-sdk/pyproject.toml >> fern/.preview/fern-python-sdk/pyproject.toml.tmp\n          mv fern/.preview/fern-python-sdk/pyproject.toml.tmp fern/.preview/fern-python-sdk/pyproject.toml\n\n          uv pip install -e fern/.preview/fern-python-sdk/.\n      - name: Migrate database\n        if: inputs.use-docker != true && inputs.test-type != 'sqlite'\n        working-directory: .\n        env:\n          LETTA_PG_PORT: 5432\n          LETTA_PG_USER: postgres\n          LETTA_PG_PASSWORD: postgres\n          LETTA_PG_DB: postgres\n          LETTA_PG_HOST: localhost\n        run: |\n          psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'\n          uv run alembic upgrade head\n      - name: Inject env vars into environment\n        if: inputs.is-external-pr != true\n        working-directory: .\n        run: |\n          # Get secrets and mask them before adding to environment\n          while IFS= read -r line || [[ -n \"$line\" ]]; do\n            if [[ -n \"$line\" ]]; then\n              value=$(echo \"$line\" | cut -d= -f2-)\n              echo \"::add-mask::$value\"\n              echo \"$line\" >> $GITHUB_ENV\n            fi\n          done < <(letta_secrets_helper --env dev --service ci)\n\n      - name: Docker setup for Docker tests\n        if: inputs.use-docker\n        run: |\n          mkdir -p /home/ci-runner/.letta/logs\n          sudo chown -R $USER:$USER /home/ci-runner/.letta/logs\n          chmod -R 755 /home/ci-runner/.letta/logs\n\n      - name: Build and run docker dev server\n        if: inputs.use-docker && inputs.is-external-pr != true\n        env:\n          LETTA_PG_DB: letta\n          LETTA_PG_USER: letta\n          LETTA_PG_PASSWORD: letta\n          LETTA_PG_PORT: 5432\n          OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }}\n        run: |\n          cd libs/config-core-deploy\n          docker compose -f compose.yaml up --build -d\n\n      - name: Wait for Docker service\n        if: inputs.use-docker\n        working-directory: ${{ steps.detect-core-dir.outputs.dir }}\n        run: |\n          bash scripts/wait_for_service.sh localhost:8083 -- echo \"Service is ready\"\n\n      - name: Run tests\n        if: inputs.is-external-pr != true\n        working-directory: ${{ steps.detect-core-dir.outputs.dir }}\n        env:\n          # Database configuration (shared, but values depend on Docker usage)\n          LETTA_PG_PORT: 5432\n          LETTA_PG_USER: ${{ inputs.use-docker && 'letta' || 'postgres' }}\n          LETTA_PG_PASSWORD: ${{ inputs.use-docker && 'letta' || 'postgres' }}\n          LETTA_PG_DB: ${{ inputs.use-docker && 'letta' || 'postgres' }}\n          LETTA_PG_HOST: localhost\n\n          # Server configuration (conditional)\n          LETTA_SERVER_PASS: test_server_token\n\n          # LLM Provider API Keys (shared across all test types)\n          OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }}\n          ANTHROPIC_API_KEY: ${{ env.ANTHROPIC_API_KEY }}\n          GEMINI_API_KEY: ${{ env.GEMINI_API_KEY }}\n          GROQ_API_KEY: ${{ env.GROQ_API_KEY }}\n          AZURE_API_KEY: ${{ env.AZURE_API_KEY }}\n          AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}\n          DEEPSEEK_API_KEY: ${{ env.DEEPSEEK_API_KEY }}\n          LETTA_MISTRAL_API_KEY: ${{ secrets.LETTA_MISTRAL_API_KEY }}\n\n          # External service API Keys (shared across all test types)\n          E2B_API_KEY: ${{ env.E2B_API_KEY }}\n          E2B_SANDBOX_TEMPLATE_ID: ${{ env.E2B_SANDBOX_TEMPLATE_ID }}\n\n          # Turbopuffer flags\n          LETTA_USE_TPUF: true\n          LETTA_TPUF_API_KEY: ${{ env.LETTA_TPUF_API_KEY }}\n\n          # Encryption key\n          LETTA_ENCRYPTION_KEY: ${{ env.LETTA_ENCRYPTION_KEY }}\n\n          # Google Cloud (shared across all test types)\n          GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT }}\n          GOOGLE_CLOUD_LOCATION: ${{ secrets.GOOGLE_CLOUD_LOCATION }}\n\n          # Real object store (required for git-backed memory integration test)\n          # Use DEV bucket/prefix variable to avoid prod resources.\n          LETTA_OBJECT_STORE_URI: ${{ vars.LETTA_OBJECT_STORE_URI_DEV }}\n\n          # Feature flags (shared across all test types)\n          LETTA_ENABLE_BATCH_JOB_POLLING: true\n\n          # Gemini flags\n          GEMINI_FORCE_MINIMUM_THINKING_BUDGET: true\n          GEMINI_MAX_RETRIES: 10\n\n          # Pinecone flags\n          LETTA_PINECONE_API_KEY: ${{ secrets.LETTA_PINECONE_API_KEY }}\n          LETTA_ENABLE_PINECONE: true\n\n          EXA_API_KEY: ${{ env.EXA_API_KEY }}\n\n          # Docker-specific environment variables\n          PYTHONPATH: ${{ inputs.use-docker && format('{0}:{1}', github.workspace, env.PYTHONPATH) || '' }}\n\n          LETTA_REDIS_HOST: localhost\n        run: |\n          set -o xtrace\n\n          # Set LETTA_SERVER_URL only for Docker tests\n          if [[ \"${{ inputs.use-docker }}\" == \"true\" ]]; then\n            export LETTA_SERVER_URL=\"http://localhost:8083\"\n          fi\n\n          # Set LLM_CONFIG_FILE only for send-message tests\n          if [[ \"${{ inputs.test-type }}\" == \"send-message\" ]]; then\n            export LLM_CONFIG_FILE=\"${{ matrix.config_file }}\"\n          fi\n\n          # Set Ollama base URL only for Ollama tests\n          if [[ \"${{ inputs.test-type }}\" == \"integration\" && \"${{ inputs.runner }}\" == *\"ollama\"* ]]; then\n            export LLM_CONFIG_FILE=\"ollama.json\"\n            export OLLAMA_BASE_URL=\"http://localhost:11434\"\n          fi\n\n          # Set LMStudio base URL only for LMStudio tests\n          if [[ \"${{ inputs.test-type }}\" == \"integration\" && \"${{ inputs.runner }}\" == *\"lmstudio\"* ]]; then\n            export LLM_CONFIG_FILE=\"lmstudio.json\"\n            export LMSTUDIO_BASE_URL=\"http://localhost:1234\"\n          fi\n\n          # Set VLLM base URL only for VLLM tests\n          if [[ \"${{ inputs.test-type }}\" == \"integration\" && \"${{ inputs.runner }}\" == *\"vllm\"* ]]; then\n            export LLM_CONFIG_FILE=\"vllm.json\"\n            export VLLM_BASE_URL=\"http://localhost:8000\"\n          fi\n\n          uv pip install pytest-github-actions-annotate-failures\n\n          # Handle different matrix variable names and test commands based on test type\n          if [[ \"${{ inputs.test-type }}\" == \"integration\" ]]; then\n            uv pip install letta\n            uv pip show letta\n            uv pip show letta-client\n            uv run --frozen pytest -svv ${{ inputs.test-path-prefix }}${{ matrix.test_suite }}\n          elif [[ \"${{ inputs.test-type }}\" == \"unit\" ]]; then\n            uv pip show letta-client\n            uv run --frozen pytest -svv ${{ inputs.test-path-prefix }}${{ matrix.test_suite }}\n          elif [[ \"${{ inputs.test-type }}\" == \"send-message\" ]]; then\n            uv run --frozen  pytest -s -vv tests/integration_test_send_message.py --maxfail=1 --durations=10\n          elif [[ \"${{ inputs.test-type }}\" == \"docker\" ]]; then\n            uv run --frozen pytest -s tests/test_client.py\n          elif [[ \"${{ inputs.test-type }}\" == \"sqlite\" ]]; then\n            # force sqlite\n            unset LETTA_PG_USER\n            unset LETTA_PG_PASSWORD\n            unset LETTA_PG_DB\n            unset LETTA_PG_HOST\n            uv pip show letta-client\n            uv run alembic upgrade head\n            uv run --frozen pytest -svv ${{ inputs.test-path-prefix }}${{ matrix.test_suite }}\n          else\n            ${{ inputs.test-command }}\n          fi\n\n      - name: Remove sqlite db\n        if: ${{ always() && inputs.test-type == 'sqlite' }}\n        run: sudo rm -rf ~/.letta || true\n\n      - name: Print docker logs if tests fail\n        if: ${{ (failure() || cancelled()) && inputs.use-docker }}\n        working-directory: libs/config-core-deploy\n        run: |\n          echo \"Printing Docker Logs...\"\n          docker compose -f compose.yaml logs\n\n      - name: Stop docker\n        if: ${{ always() && inputs.use-docker }}\n        working-directory: libs/config-core-deploy\n        run: |\n          docker compose -f compose.yaml down --volumes\n          sudo rm -rf .persist\n"}}}}