hack/tools/scripts/llm-key-hunter/results/_cache/content/benchflow-ai_benchflow.json

{"repo": "benchflow-ai/benchflow", "files": {".github/workflows/integration-final-review.yml": {"cf656179f3bc30b2887b744cf71b69b5e5ffa5ff": {"t": 1785745246.8221622, "n": 31228, "h": "534fd04c7fee20cf", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          # Leave the job a few minutes to upload artifacts and run cleanup after\n          # the subprocess times out; otherwise GitHub can kill the whole job first.\n          subprocess_timeout = max(1.0, float(cell.get(\"timeout_minutes\", 90)) - 5.0) * 60\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n              timeout=subprocess_timeout,\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # Route codex to DeepSeek-v4-pro via Moon Bridge (a local OpenAI-Responses\n      # <-> DeepSeek forwarding proxy) — codex speaks ONLY the Responses API and\n      # the OpenAI account quota is exhausted, so the reviewer runs on DeepSeek.\n      # Starts the bridge in the background + writes codex's config.toml. Method:\n      # github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md.\n      - name: Set up codex DeepSeek bridge (Moon Bridge)\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n        run: bash .github/scripts/setup_codex_deepseek_bridge.sh\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # codex runs on DeepSeek-v4-pro through Moon Bridge (config.toml written\n          # by the setup step into ~/.codex, model `moonbridge` -> deepseek-v4-pro).\n          # CODEX_API_KEY is still set so has_codex_auth passes; the moonbridge\n          # provider is a local bridge and ignores it.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          CODEX_MODEL: moonbridge\n          CODEX_REASONING_EFFORT: xhigh\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          # The bridge setup step is continue-on-error, so if it failed there is no\n          # ~/.codex/config.toml and codex would only emit a confusing\n          # connection-refused against the dead local proxy. Detect that absence up\n          # front and fail-closed with a clear message instead.\n          if [ ! -f \"${CODEX_HOME:-$HOME/.codex}/config.toml\" ]; then\n            echo \"::error title=Moon Bridge unavailable::codex DeepSeek bridge setup did not complete (no config.toml) — codex (required at L3) cannot run; fail-closing to 'not mergeable (codex unavailable)'.\"\n            echo \"codex_exit=1\" >> \"$GITHUB_OUTPUT\"\n          else\n            uv run python .github/scripts/codex_review.py \\\n              --review-pack review-pack \\\n              --artifacts jobs/integration-final \\\n              --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n              --findings-out review-pack/codex_findings.json \\\n              --codex-out review-pack/codex_output.txt\n            rc=$?\n            echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          fi\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          # The fail-closed trigger is an empty CODEX_FINAL (codex produced no\n          # verdict). codex_exit is intentionally not consulted: codex_review.py\n          # returns non-zero for a legitimately-computed \"not mergeable\" too, so\n          # gating on it would mislabel a real verdict as \"codex unavailable\".\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "6ef06bcbf909551e803b13ad2f3991f9cfccb529": {"t": 1785749198.3563507, "n": 31228, "h": "534fd04c7fee20cf", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          # Leave the job a few minutes to upload artifacts and run cleanup after\n          # the subprocess times out; otherwise GitHub can kill the whole job first.\n          subprocess_timeout = max(1.0, float(cell.get(\"timeout_minutes\", 90)) - 5.0) * 60\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n              timeout=subprocess_timeout,\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # Route codex to DeepSeek-v4-pro via Moon Bridge (a local OpenAI-Responses\n      # <-> DeepSeek forwarding proxy) — codex speaks ONLY the Responses API and\n      # the OpenAI account quota is exhausted, so the reviewer runs on DeepSeek.\n      # Starts the bridge in the background + writes codex's config.toml. Method:\n      # github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md.\n      - name: Set up codex DeepSeek bridge (Moon Bridge)\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n        run: bash .github/scripts/setup_codex_deepseek_bridge.sh\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # codex runs on DeepSeek-v4-pro through Moon Bridge (config.toml written\n          # by the setup step into ~/.codex, model `moonbridge` -> deepseek-v4-pro).\n          # CODEX_API_KEY is still set so has_codex_auth passes; the moonbridge\n          # provider is a local bridge and ignores it.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          CODEX_MODEL: moonbridge\n          CODEX_REASONING_EFFORT: xhigh\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          # The bridge setup step is continue-on-error, so if it failed there is no\n          # ~/.codex/config.toml and codex would only emit a confusing\n          # connection-refused against the dead local proxy. Detect that absence up\n          # front and fail-closed with a clear message instead.\n          if [ ! -f \"${CODEX_HOME:-$HOME/.codex}/config.toml\" ]; then\n            echo \"::error title=Moon Bridge unavailable::codex DeepSeek bridge setup did not complete (no config.toml) — codex (required at L3) cannot run; fail-closing to 'not mergeable (codex unavailable)'.\"\n            echo \"codex_exit=1\" >> \"$GITHUB_OUTPUT\"\n          else\n            uv run python .github/scripts/codex_review.py \\\n              --review-pack review-pack \\\n              --artifacts jobs/integration-final \\\n              --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n              --findings-out review-pack/codex_findings.json \\\n              --codex-out review-pack/codex_output.txt\n            rc=$?\n            echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          fi\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          # The fail-closed trigger is an empty CODEX_FINAL (codex produced no\n          # verdict). codex_exit is intentionally not consulted: codex_review.py\n          # returns non-zero for a legitimately-computed \"not mergeable\" too, so\n          # gating on it would mislabel a real verdict as \"codex unavailable\".\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "6f023e05819aa391092d548bbf9f5925651d6365": {"t": 1785749201.7815866, "n": 30928, "h": "c165b5e6389a5869", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # Route codex to DeepSeek-v4-pro via Moon Bridge (a local OpenAI-Responses\n      # <-> DeepSeek forwarding proxy) — codex speaks ONLY the Responses API and\n      # the OpenAI account quota is exhausted, so the reviewer runs on DeepSeek.\n      # Starts the bridge in the background + writes codex's config.toml. Method:\n      # github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md.\n      - name: Set up codex DeepSeek bridge (Moon Bridge)\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n        run: bash .github/scripts/setup_codex_deepseek_bridge.sh\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # codex runs on DeepSeek-v4-pro through Moon Bridge (config.toml written\n          # by the setup step into ~/.codex, model `moonbridge` -> deepseek-v4-pro).\n          # CODEX_API_KEY is still set so has_codex_auth passes; the moonbridge\n          # provider is a local bridge and ignores it.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          CODEX_MODEL: moonbridge\n          CODEX_REASONING_EFFORT: xhigh\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          # The bridge setup step is continue-on-error, so if it failed there is no\n          # ~/.codex/config.toml and codex would only emit a confusing\n          # connection-refused against the dead local proxy. Detect that absence up\n          # front and fail-closed with a clear message instead.\n          if [ ! -f \"${CODEX_HOME:-$HOME/.codex}/config.toml\" ]; then\n            echo \"::error title=Moon Bridge unavailable::codex DeepSeek bridge setup did not complete (no config.toml) — codex (required at L3) cannot run; fail-closing to 'not mergeable (codex unavailable)'.\"\n            echo \"codex_exit=1\" >> \"$GITHUB_OUTPUT\"\n          else\n            uv run python .github/scripts/codex_review.py \\\n              --review-pack review-pack \\\n              --artifacts jobs/integration-final \\\n              --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n              --findings-out review-pack/codex_findings.json \\\n              --codex-out review-pack/codex_output.txt\n            rc=$?\n            echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          fi\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          # The fail-closed trigger is an empty CODEX_FINAL (codex produced no\n          # verdict). codex_exit is intentionally not consulted: codex_review.py\n          # returns non-zero for a legitimately-computed \"not mergeable\" too, so\n          # gating on it would mislabel a real verdict as \"codex unavailable\".\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "3daf698643c7db5fecf1432da3fb1bf2c837e46c": {"t": 1785749203.0975418, "n": 29989, "h": "bfaa800182657770", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # Route codex to DeepSeek-v4-pro via Moon Bridge (a local OpenAI-Responses\n      # <-> DeepSeek forwarding proxy) — codex speaks ONLY the Responses API and\n      # the OpenAI account quota is exhausted, so the reviewer runs on DeepSeek.\n      # Starts the bridge in the background + writes codex's config.toml. Method:\n      # github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md.\n      - name: Set up codex DeepSeek bridge (Moon Bridge)\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n        run: bash .github/scripts/setup_codex_deepseek_bridge.sh\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # codex runs on DeepSeek-v4-pro through Moon Bridge (config.toml written\n          # by the setup step into ~/.codex, model `moonbridge` -> deepseek-v4-pro).\n          # CODEX_API_KEY is still set so has_codex_auth passes; the moonbridge\n          # provider is a local bridge and ignores it.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          CODEX_MODEL: moonbridge\n          CODEX_REASONING_EFFORT: xhigh\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          uv run python .github/scripts/codex_review.py \\\n            --review-pack review-pack \\\n            --artifacts jobs/integration-final \\\n            --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n            --findings-out review-pack/codex_findings.json \\\n            --codex-out review-pack/codex_output.txt\n          rc=$?\n          echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n          CODEX_EXIT: ${{ steps.codex.outputs.codex_exit }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "9db14f9187e883fe2877e6111d50d1350f0e7531": {"t": 1785749205.3488214, "n": 29755, "h": "1ef5604cd8ba3fc3", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # The per-rollout judge (Pass 1) uses the DeepSeek OPENAI_* that\n          # provider-select wrote to $GITHUB_ENV. The host `codex exec` (Pass 2)\n          # needs the REAL OpenAI key — passed via CODEX_API_KEY, which\n          # codex_review.py uses for the CLI (dropping the DeepSeek base URL) so\n          # the judge clobber never leaks into codex auth.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          # codex speaks ONLY the OpenAI Responses API — DeepSeek is impossible\n          # (codex removed chat-wire; DeepSeek doesn't serve Responses). So the\n          # composer runs on an OpenAI model: gpt-5.5 (full model — supports\n          # codex's tools, unlike gpt-5.4-nano which rejected `tool_search`) with\n          # xhigh reasoning effort for a thorough equivalence review.\n          CODEX_MODEL: gpt-5.5\n          CODEX_REASONING_EFFORT: xhigh\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          uv run python .github/scripts/codex_review.py \\\n            --review-pack review-pack \\\n            --artifacts jobs/integration-final \\\n            --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n            --findings-out review-pack/codex_findings.json \\\n            --codex-out review-pack/codex_output.txt\n          rc=$?\n          echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n          CODEX_EXIT: ${{ steps.codex.outputs.codex_exit }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "4fe3f39cda0764ea595603641e5083139a010064": {"t": 1785749206.6162667, "n": 29653, "h": "b13a8122efa9f84c", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # The per-rollout judge (Pass 1) uses the DeepSeek OPENAI_* that\n          # provider-select wrote to $GITHUB_ENV. The host `codex exec` (Pass 2)\n          # needs the REAL OpenAI key — passed via CODEX_API_KEY, which\n          # codex_review.py uses for the CLI (dropping the DeepSeek base URL) so\n          # the judge clobber never leaks into codex auth.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          # Pin the codex composer to a model the OPENAI_API_KEY is PROVEN to serve\n          # (the codex-acp matrix cells use gpt-5.4-nano). Otherwise `codex exec`\n          # falls back to its built-in default (gpt-5.x-codex), which the key may\n          # not be entitled to -> model_not_found -> 'codex unavailable'.\n          CODEX_MODEL: gpt-5.4-nano\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          uv run python .github/scripts/codex_review.py \\\n            --review-pack review-pack \\\n            --artifacts jobs/integration-final \\\n            --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n            --findings-out review-pack/codex_findings.json \\\n            --codex-out review-pack/codex_output.txt\n          rc=$?\n          echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n          CODEX_EXIT: ${{ steps.codex.outputs.codex_exit }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "585efc53a0c6e583729df4f3ddd460af80c9f9c2": {"t": 1785749208.9239268, "n": 29295, "h": "81d1344c09a6bbd5", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # The per-rollout judge (Pass 1) uses the DeepSeek OPENAI_* that\n          # provider-select wrote to $GITHUB_ENV. The host `codex exec` (Pass 2)\n          # needs the REAL OpenAI key — passed via CODEX_API_KEY, which\n          # codex_review.py uses for the CLI (dropping the DeepSeek base URL) so\n          # the judge clobber never leaks into codex auth.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          uv run python .github/scripts/codex_review.py \\\n            --review-pack review-pack \\\n            --artifacts jobs/integration-final \\\n            --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n            --findings-out review-pack/codex_findings.json \\\n            --codex-out review-pack/codex_output.txt\n          rc=$?\n          echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n          CODEX_EXIT: ${{ steps.codex.outputs.codex_exit }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "5a03e96e821f65f03875d317d186681ac37b51e7": {"t": 1785749210.179349, "n": 28692, "h": "d13427824763c4d6", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # Codex auth = OPENAI_API_KEY from the job env (codex_review.py writes\n          # it as an apikey auth.json). BENCHFLOW_JUDGE_MODEL is already in the\n          # job env (provider-select wrote it to $GITHUB_ENV) for the cheap\n          # deepseek per-rollout pass.\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          uv run python .github/scripts/codex_review.py \\\n            --review-pack review-pack \\\n            --artifacts jobs/integration-final \\\n            --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n            --findings-out review-pack/codex_findings.json \\\n            --codex-out review-pack/codex_output.txt\n          rc=$?\n          echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n          CODEX_EXIT: ${{ steps.codex.outputs.codex_exit }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "3b4e6b074aff8336a9ad635589d353f21c032ddd": {"t": 1785749212.4548345, "n": 28617, "h": "440b684da97e5feb", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter — the planner itself\n          # is pure stdlib and needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # Codex auth = OPENAI_API_KEY from the job env (codex_review.py writes\n          # it as an apikey auth.json). BENCHFLOW_JUDGE_MODEL is already in the\n          # job env (provider-select wrote it to $GITHUB_ENV) for the cheap\n          # deepseek per-rollout pass.\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          uv run python .github/scripts/codex_review.py \\\n            --review-pack review-pack \\\n            --artifacts jobs/integration-final \\\n            --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n            --findings-out review-pack/codex_findings.json \\\n            --codex-out review-pack/codex_output.txt\n          rc=$?\n          echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n          CODEX_EXIT: ${{ steps.codex.outputs.codex_exit }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "e1060f6cc3eca6226f60971be9b2dc340763c15a": {"t": 1785749213.6828225, "n": 26645, "h": "6da7cebe4778ce33", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * NEVER on push / pull_request: this workflow is workflow_dispatch ONLY. The\n#     jobs run under the existing 'pypi-internal-preview' environment (which holds\n#     the provider + DAYTONA secrets). No separate protected env is created — the\n#     L3 golden truth is the HuggingFace leaderboard `main` runs (the deepseek-\n#     v4-flash baseline) compared against the PR vs the latest benchflow main.\n#     (To add a human-approval gate later, point these jobs at a protected env.)\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          sparse-checkout: |\n            .github\n            tests/integration\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        run: pip install --quiet .\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # Codex auth = OPENAI_API_KEY from the job env (codex_review.py writes\n          # it as an apikey auth.json). BENCHFLOW_JUDGE_MODEL is already in the\n          # job env (provider-select wrote it to $GITHUB_ENV) for the cheap\n          # deepseek per-rollout pass.\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          uv run python .github/scripts/codex_review.py \\\n            --review-pack review-pack \\\n            --artifacts jobs/integration-final \\\n            --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n            --findings-out review-pack/codex_findings.json \\\n            --codex-out review-pack/codex_output.txt\n          rc=$?\n          echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n          CODEX_EXIT: ${{ steps.codex.outputs.codex_exit }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}, "2631b1d47436fbaa6017956c2957e7d4147b3cae": {"t": 1786068220.9639506, "n": 31228, "h": "534fd04c7fee20cf", "c": "# integration-final-review — L3: the manual, golden-truth FINAL review.\n#\n# SECURITY MODEL (read before editing):\n#   * TRUSTED-MAIN INFRA: the planner (integration_matrix.py), the grader\n#     (build_integration_review_pack.py), the harness (scenarios), and the codex\n#     reviewer (codex_review.py) ALWAYS load from origin/main via sparse checkout.\n#     Only src/benchflow (the code-under-test) is overlaid from the PR head SHA.\n#   * TRIGGERS: workflow_dispatch (manual, any PR) OR a `pull_request` `labeled`\n#     event when the `ready-to-merge` label is added — the auto path: marking a PR\n#     ready to merge runs the FULL final review, codex included. NEVER on push and\n#     NEVER pull_request_target, so fork PRs receive no secrets; the plan job's\n#     resolve step additionally refuses any non-internal head repo, and only\n#     write-access users can apply labels. The jobs run under the existing\n#     'pypi-internal-preview' environment (provider + DAYTONA secrets). No separate\n#     protected env — the L3 golden truth is the HuggingFace leaderboard `main`\n#     runs (deepseek-v4-flash baseline) vs the PR and the latest benchflow main.\n#   * FAIL-CLOSED CODEX: at L3 codex_review.py is REQUIRED. A missing codex\n#     binary / codex auth (the repo OPENAI_API_KEY), or unparseable codex output, yields\n#     'not mergeable (codex unavailable)' and a red check. The published\n#     'integration-final-review' check-run conclusion = worst(deterministic,\n#     codex): codex can only make the verdict STRICTER, never upgrade a\n#     deterministic 'not mergeable'.\n#   * concurrency is keyed per pr_number with cancel-in-progress: FALSE — a real\n#     multi-cell rollout that has burned credits must finish, not be cancelled.\nname: integration-final-review\n\non:\n  workflow_dispatch:\n    inputs:\n      pr_number:\n        description: PR number to run the final review against.\n        required: true\n        type: string\n      scope:\n        description: Scope (auto|citation|low-3|medium-3|high-3|nine|expanded|custom).\n        required: false\n        type: string\n        default: auto\n      custom_tasks:\n        description: Comma-separated task ids for scope=custom.\n        required: false\n        type: string\n        default: \"\"\n      sandboxes:\n        description: Comma-separated sandbox override (docker,daytona).\n        required: false\n        type: string\n        default: docker\n      skill_modes:\n        description: Comma-separated skill-mode override (no-skill,with-skill,self-gen).\n        required: false\n        type: string\n        default: no-skill\n      codex_review:\n        description: Run the codex equivalence reviewer (required gate at L3).\n        required: false\n        type: boolean\n        default: true\n\n  # Auto path: adding the `ready-to-merge` label to an INTERNAL PR runs the full\n  # final review (codex included). Plain pull_request (NEVER pull_request_target),\n  # so fork PRs receive no secrets; the plan job's resolve step additionally\n  # refuses any non-internal head repo, and only write-access users can label.\n  pull_request:\n    types: [labeled]\n\n# One final review per PR at a time; do NOT cancel an in-flight rollout.\nconcurrency:\n  group: integration-final-review-${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n  cancel-in-progress: false\n\npermissions:\n  contents: read\n  pull-requests: read\n  actions: read\n  models: read\n\njobs:\n  # ----------------------------------------------------------------------------\n  # plan — TRUSTED main. Resolves the PR head SHA via the API, re-asserts the PR\n  # is internal, and runs the planner -> matrix + head_sha outputs. Fails on the\n  # planner's exit-2 hard-ceiling overflow.\n  # ----------------------------------------------------------------------------\n  plan:\n    runs-on: ubuntu-latest\n    # Manual dispatch (any PR) OR the `ready-to-merge` auto-label trigger. Any\n    # OTHER label added to a PR is a no-op: this job — and the run-matrix /\n    # review-pack jobs that depend on its outputs — are skipped, so an unrelated\n    # label never spins up a rollout and the workflow stays green.\n    if: >-\n      github.event_name == 'workflow_dispatch' ||\n      (github.event_name == 'pull_request' && github.event.label.name == 'ready-to-merge')\n    # Run under the same env that run-matrix uses so the credential filter\n    # below can see the real provider secrets (DEEPSEEK/GEMINI/OPENAI/Bedrock).\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: read\n    outputs:\n      matrix: ${{ steps.emit.outputs.matrix }}\n      head_sha: ${{ steps.resolve.outputs.head_sha }}\n      has_cells: ${{ steps.emit.outputs.has_cells }}\n      base_ref: ${{ steps.resolve.outputs.base_ref }}\n    steps:\n      - name: Resolve + authorize PR\n        id: resolve\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n        run: |\n          set -euo pipefail\n          json=$(gh api \"repos/${REPO}/pulls/${PR_NUMBER}\")\n          head_repo=$(printf '%s' \"$json\" | jq -r '.head.repo.full_name // \"\"')\n          if [ \"$head_repo\" != \"benchflow-ai/benchflow\" ]; then\n            echo \"::error::PR #${PR_NUMBER} head repo is '${head_repo}', not internal — refusing.\"\n            exit 1\n          fi\n          head_sha=$(printf '%s' \"$json\" | jq -r '.head.sha')\n          base_ref=$(printf '%s' \"$json\" | jq -r '.base.ref')\n          echo \"head_sha=$head_sha\" >> \"$GITHUB_OUTPUT\"\n          echo \"base_ref=$base_ref\" >> \"$GITHUB_OUTPUT\"\n          echo \"Resolved PR #${PR_NUMBER}: head=$head_sha base=$base_ref\"\n\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ steps.resolve.outputs.base_ref }}\n          fetch-depth: 0\n          # pyproject.toml + README.md + src so `uv pip install .` below can build\n          # benchflow (hatchling) for the credential filter; uv.lock so the\n          # setup-uv cache key resolves. The planner itself is pure stdlib and\n          # needs only .github.\n          sparse-checkout: |\n            .github\n            tests/integration\n            pyproject.toml\n            uv.lock\n            README.md\n            src\n          sparse-checkout-cone-mode: false\n\n      - name: Fetch PR head SHA for the diff\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n        run: git fetch --no-tags origin \"$HEAD_SHA\"\n\n      - name: Set up Python\n        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0\n        with:\n          python-version: \"3.12\"\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Plan final-review scope (trusted)\n        id: plan\n        env:\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          BASE_REF: ${{ steps.resolve.outputs.base_ref }}\n          HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          CUSTOM_TASKS: ${{ github.event.inputs.custom_tasks || '' }}\n          SANDBOXES: ${{ github.event.inputs.sandboxes || 'docker' }}\n          SKILL_MODES: ${{ github.event.inputs.skill_modes || 'no-skill' }}\n        run: |\n          set -euo pipefail\n          ARGS=( --pr-number \"$PR_NUMBER\" --base-ref \"$BASE_REF\" --head-sha \"$HEAD_SHA\"\n                 --diff-from-git --scope \"$SCOPE_INPUT\" --out matrix.json )\n          if [ -n \"$CUSTOM_TASKS\" ]; then ARGS+=( --custom-tasks \"$CUSTOM_TASKS\" ); fi\n          if [ -n \"$SANDBOXES\" ]; then ARGS+=( --sandboxes \"$SANDBOXES\" ); fi\n          if [ -n \"$SKILL_MODES\" ]; then ARGS+=( --skill-modes \"$SKILL_MODES\" ); fi\n          set +e\n          python .github/scripts/integration_matrix.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          if [ \"$rc\" -eq 2 ]; then\n            REASON=$(jq -r '.rejected_overflow // \"scope overflow\"' matrix.json 2>/dev/null || echo \"scope overflow\")\n            echo \"::error::integration scope overflow: $REASON\"\n            exit 1\n          elif [ \"$rc\" -ne 0 ]; then\n            echo \"::error::scope planner failed (exit $rc)\"\n            exit \"$rc\"\n          fi\n          # matrix output is emitted AFTER the credential filter below.\n          echo \"Planned scope=$(jq -r '.scope' matrix.json) cells=$(jq '.matrix | length' matrix.json)\"\n\n      # Install benchflow so the credential filter can import\n      # benchflow.agents.env. The planner above is pure (MiniYaml-safe) and\n      # needs nothing; only this env-aware step needs the package.\n      - name: Install benchflow (for the credential filter)\n        # uv pip install into the setup-python interpreter (--system). The filter\n        # is fail-open by design, so a failed install must NOT block the plan —\n        # warn and let the filter pass the matrix through unfiltered.\n        run: uv pip install --system --quiet . || echo \"::warning::benchflow install failed; credential filter will fail-open (matrix unfiltered)\"\n\n      # Drop any planned cell whose agent+model credential is absent from this\n      # environment BEFORE run-matrix consumes the matrix, so un-keyed agents\n      # never spin up a (doomed, sandbox-burning) cell that would log as a\n      # false-red slot. Fails OPEN: if benchflow import fails the matrix passes\n      # through unchanged. Credentials come from the pypi-internal-preview env.\n      - name: Filter uncredentialed cells (env-aware)\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n          DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}\n          AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}\n        run: |\n          set -euo pipefail\n          python .github/scripts/filter_credentialed_cells.py \\\n            --matrix matrix.json --out matrix.json\n          SKIPPED=$(jq -c '.skipped_uncredentialed // []' matrix.json)\n          if [ \"$SKIPPED\" != \"[]\" ]; then\n            echo \"::notice::credential filter skipped uncredentialed cells: $SKIPPED\"\n          fi\n\n      - name: Emit filtered matrix output\n        id: emit\n        run: |\n          set -euo pipefail\n          N=$(jq '.matrix | length' matrix.json)\n          echo \"matrix=$(jq -c '.matrix' matrix.json)\" >> \"$GITHUB_OUTPUT\"\n          echo \"Run matrix cells=$N (after credential filter)\"\n          # Empty matrix after credential filtering (NO provider key for ANY\n          # planned cell) must SKIP run-matrix/review-pack as a green no-op — NOT\n          # hard-fail fromJson([]) (\"matrix must define at least one combination\")\n          # into a false 'not mergeable' verdict on a manual final-review dispatch.\n          if [ \"$N\" -eq 0 ]; then\n            echo \"has_cells=false\" >> \"$GITHUB_OUTPUT\"\n            echo \"::warning::all planned cells filtered out (no provider credentials available) — passing as a green no-op; nothing ran\"\n          else\n            echo \"has_cells=true\" >> \"$GITHUB_OUTPUT\"\n          fi\n\n      - name: Upload matrix plan\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-matrix\n          path: matrix.json\n          if-no-files-found: error\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # run-matrix — PROTECTED environment (required reviewers = the human L3 gate).\n  # One leg per planned cell; per-cell timeout from the plan. Trusted-main harness\n  # + PR-head src/benchflow. All provider + DAYTONA keys. usage-tracking required;\n  # agent_judge per cell; skill-catalog audit when cell.audit_skills.\n  # ----------------------------------------------------------------------------\n  run-matrix:\n    needs: plan\n    if: needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n    strategy:\n      fail-fast: false\n      matrix:\n        cell: ${{ fromJson(needs.plan.outputs.matrix) }}\n    timeout-minutes: ${{ matrix.cell.timeout_minutes }}\n    env:\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      GLM_API_KEY: ${{ secrets.GLM_API_KEY }}\n      GLM_BASE_URL: ${{ secrets.GLM_BASE_URL }}\n      QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}\n      QWEN_BASE_URL: ${{ secrets.QWEN_BASE_URL }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}\n      CELL_JSON: ${{ toJson(matrix.cell) }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Overlay PR-head src/benchflow (code-under-test)\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n        run: |\n          set -euo pipefail\n          git fetch --no-tags origin \"$HEAD_SHA\"\n          git checkout \"$HEAD_SHA\" -- src/benchflow\n\n      - name: Select integration provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --extra sandbox-daytona --locked\n\n      - name: Run the cell (trusted-main scenarios.run_eval)\n        env:\n          # The provider-select step wrote OPENAI_*=DeepSeek into the job env for\n          # the DeepSeek agent-judge. The gated-native codex-acp rollout (gpt,\n          # OpenAI Responses) KEEPS ambient OPENAI_* (it does not recompute them\n          # in resolve_agent_env), so it would otherwise auth its rollout against\n          # DeepSeek and fail. Restore the REAL OpenAI creds for codex-acp cells\n          # ONLY — every DeepSeek cell keeps the job env unchanged (no-op), and the\n          # later agent-judge step keeps the DeepSeek OPENAI_* from $GITHUB_ENV.\n          OPENAI_API_KEY: ${{ matrix.cell.agent == 'codex-acp' && secrets.OPENAI_API_KEY || env.OPENAI_API_KEY }}\n          OPENAI_BASE_URL: ${{ matrix.cell.agent == 'codex-acp' && 'https://api.openai.com/v1' || env.OPENAI_BASE_URL }}\n        run: |\n          uv run python - <<'PY'\n          import json\n          import os\n          import pathlib\n          import sys\n\n          sys.path.insert(0, \"tests/integration\")\n          import scenarios\n\n          cell = json.loads(os.environ[\"CELL_JSON\"])\n          cid = cell[\"id\"]\n          jobs_dir = pathlib.Path(\"jobs/integration-final\") / cid\n          jobs_dir.mkdir(parents=True, exist_ok=True)\n\n          extra = [\n              \"--skill-mode\", cell.get(\"skill_mode\", \"no-skill\"),\n              \"--usage-tracking\", \"required\",\n              \"--agent-idle-timeout\", str(cell.get(\"agent_idle_timeout\", 240)),\n          ]\n          # Leave the job a few minutes to upload artifacts and run cleanup after\n          # the subprocess times out; otherwise GitHub can kill the whole job first.\n          subprocess_timeout = max(1.0, float(cell.get(\"timeout_minutes\", 90)) - 5.0) * 60\n          outcome = scenarios.run_eval(\n              jobs_dir=jobs_dir,\n              agent=cell[\"agent\"],\n              sandbox=cell.get(\"sandbox\", \"docker\"),\n              model=cell.get(\"model\") or None,\n              include=(cell[\"task\"],),\n              concurrency=1,\n              extra_args=tuple(extra),\n              source_repo=\"benchflow-ai/skillsbench\",\n              source_path=\"tasks\",\n              source_ref=\"main\",\n              timeout=subprocess_timeout,\n          )\n          print(f\"cell={cid} rc={outcome.returncode} jobs_dir={jobs_dir}\")\n          PY\n\n      - name: Agent-judge the cell\n        run: |\n          uv run python tests/integration/agent_judge.py \\\n            \"jobs/integration-final/$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\" \\\n            --model \"$BENCHFLOW_JUDGE_MODEL\" \\\n            --json || echo \"::notice::agent_judge non-zero (graded downstream by review-pack)\"\n\n      - name: Skill-catalog audit (when the cell requests it)\n        # Best-effort pre-extraction; the authoritative S-WITHSKILL gate runs in\n        # the trusted-main grader (rubric_checks via build_integration_review_pack).\n        # The extractor is the standalone CLI shipped in the experiment-review\n        # skill tree (positional trajectory path).\n        if: ${{ matrix.cell.audit_skills }}\n        env:\n          EXTRACT: .agents/skills/benchflow-experiment-review/scripts/extract_harness_skills.py\n        run: |\n          set -euo pipefail\n          CID=$(uv run python -c \"import json,os;print(json.loads(os.environ['CELL_JSON'])['id'])\")\n          while IFS= read -r -d '' traj; do\n            uv run python \"$EXTRACT\" \"$traj\" || \\\n              echo \"::notice::skill catalog extraction flagged $traj (quarantine downstream)\"\n          done < <(find \"jobs/integration-final/$CID\" -path '*/trajectory/llm_trajectory.jsonl' -print0)\n\n      - name: Upload cell jobs\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-${{ strategy.job-index }}\n          path: jobs/integration-final\n          if-no-files-found: ignore\n          retention-days: 30\n\n  # ----------------------------------------------------------------------------\n  # review-pack — TRUSTED grader + REQUIRED codex. always() so it grades whatever\n  # ran. Builds the deterministic review pack (pinned baseline; rerun-base only\n  # for scope=expanded), then runs codex_review.py as the REQUIRED fail-closed\n  # gate. Sets the 'integration-final-review' check-run conclusion =\n  # worst(deterministic, codex) and upserts the sticky verdict comment.\n  # ----------------------------------------------------------------------------\n  review-pack:\n    needs: [plan, run-matrix]\n    if: always() && needs.plan.outputs.has_cells == 'true'\n    runs-on: ubuntu-latest\n    environment: pypi-internal-preview\n    permissions:\n      contents: read\n      pull-requests: write\n      checks: write\n    env:\n      LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY || secrets.BF_TOKEN }}\n      LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL || 'http://llm-proxy.eval.all-hands.dev/v1' }}\n      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n      GITHUB_MODELS_TOKEN: ${{ secrets.GITHUB_MODELS_TOKEN || github.token }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      DEEPSEEK_BASE_URL: ${{ secrets.DEEPSEEK_BASE_URL }}\n      # L3 Codex reviewer auth: the existing repo-level OPENAI_API_KEY (durable,\n      # revocable) — codex_review.py writes it as an apikey auth.json. No\n      # personal-OAuth CODEX_AUTH_JSON blob is needed.\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n    steps:\n      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n        with:\n          ref: ${{ needs.plan.outputs.base_ref }}\n          fetch-depth: 0\n\n      - name: Download matrix plan\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        with:\n          name: integration-final-matrix\n          path: .\n\n      - name: Download all cell jobs\n        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0\n        continue-on-error: true\n        with:\n          pattern: integration-final-*\n          merge-multiple: true\n          path: jobs/integration-final\n\n      - name: Select judge provider\n        run: python .github/scripts/select_integration_provider.py\n\n      - name: Install uv\n        uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39  # v3.2.4\n        with:\n          enable-cache: true\n\n      - name: Set up Python\n        run: uv python install 3.12\n\n      - name: Install dependencies\n        run: uv sync --extra dev --extra judge --locked\n\n      - name: Fetch leaderboard golden-truth baseline\n        # Pinned-baseline reward-band parity (Q2b). Downloads the matching-config\n        # SkillsBench leaderboard submissions (native BenchFlow result.json) from\n        # the HF dataset + revision recorded in scope_defaults.yml -> baseline.\n        # The band model covers every integration task; cheap deepseek runs band-\n        # compare advisory-only. Best-effort: a fetch miss leaves band parity NA\n        # rather than failing the gate (the deterministic floor + codex still run).\n        id: baseline\n        continue-on-error: true\n        run: |\n          set -euo pipefail\n          vals=$(uv run python -c \"import yaml;d=yaml.safe_load(open('.github/integration/scope_defaults.yml'))['baseline'];print(d['hf_repo'],d['hf_revision'],d['submission_root'],d['config_no_skill'],d['band_model'])\")\n          read -r HF_REPO HF_REV SUB_ROOT CONFIG BAND_MODEL <<< \"$vals\"\n          DIR=\"$SUB_ROOT/${CONFIG}__${BAND_MODEL}\"\n          uvx --from huggingface_hub hf download \"$HF_REPO\" --repo-type dataset \\\n            --revision \"$HF_REV\" --include \"$DIR/**/result.json\" \\\n            --local-dir baseline-root\n          echo \"baseline_dir=baseline-root/$DIR\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Build the deterministic review pack\n        id: pack\n        env:\n          SCOPE_INPUT: ${{ github.event.inputs.scope || 'auto' }}\n          BASELINE_DIR: ${{ steps.baseline.outputs.baseline_dir }}\n        run: |\n          set -euo pipefail\n          ARGS=( --matrix matrix.json --artifacts jobs/integration-final --out review-pack )\n          # Pinned leaderboard baseline (golden truth) is the default vehicle: the\n          # fetch step exported BASELINE_DIR. scope=expanded additionally allows a\n          # same-SHA base re-run via harbor-baseline.\n          if [ -n \"${BASELINE_DIR:-}\" ] && [ -d \"${BASELINE_DIR}\" ]; then\n            ARGS+=( --baseline-root \"${BASELINE_DIR}\" )\n          elif [ \"$SCOPE_INPUT\" = \"expanded\" ] && [ -d harbor-baseline ]; then\n            ARGS+=( --baseline-root harbor-baseline )\n          fi\n          set +e\n          uv run python .github/scripts/build_integration_review_pack.py \"${ARGS[@]}\"\n          rc=$?\n          set -e\n          echo \"pack_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          DET=\"not mergeable\"\n          if [ -f review-pack/verdict.md ]; then\n            DET=$(grep -ioE 'mergeable with quarantines|not mergeable|mergeable' review-pack/verdict.md | head -1 || echo \"not mergeable\")\n          else\n            mkdir -p review-pack\n            printf '## Verdict\\n\\nnot mergeable\\n\\nThe review pack failed before writing a verdict (exit %s).\\n' \"$rc\" > review-pack/verdict.md\n          fi\n          echo \"deterministic_verdict=$DET\" >> \"$GITHUB_OUTPUT\"\n          echo \"deterministic verdict: $DET (pack exit $rc)\"\n\n      # The host codex CLI (`codex exec`) the equivalence reviewer drives.\n      # continue-on-error so a failed install reaches the codex step, which then\n      # fail-closes to 'not mergeable (codex unavailable)' with a verdict, rather\n      # than dying as a raw step error.\n      - name: Install codex CLI\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        run: npm install -g @openai/codex@0.141.0\n\n      # Route codex to DeepSeek-v4-pro via Moon Bridge (a local OpenAI-Responses\n      # <-> DeepSeek forwarding proxy) — codex speaks ONLY the Responses API and\n      # the OpenAI account quota is exhausted, so the reviewer runs on DeepSeek.\n      # Starts the bridge in the background + writes codex's config.toml. Method:\n      # github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md.\n      - name: Set up codex DeepSeek bridge (Moon Bridge)\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        continue-on-error: true\n        env:\n          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n        run: bash .github/scripts/setup_codex_deepseek_bridge.sh\n\n      # REQUIRED at L3 (fail-closed). A dead/absent codex key => not mergeable.\n      - name: Codex equivalence review (REQUIRED, fail-closed)\n        id: codex\n        if: ${{ github.event.inputs.codex_review != 'false' }}\n        env:\n          # codex runs on DeepSeek-v4-pro through Moon Bridge (config.toml written\n          # by the setup step into ~/.codex, model `moonbridge` -> deepseek-v4-pro).\n          # CODEX_API_KEY is still set so has_codex_auth passes; the moonbridge\n          # provider is a local bridge and ignores it.\n          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n          CODEX_MODEL: moonbridge\n          CODEX_REASONING_EFFORT: xhigh\n          DETERMINISTIC_VERDICT: ${{ steps.pack.outputs.deterministic_verdict }}\n        run: |\n          set +e\n          # The bridge setup step is continue-on-error, so if it failed there is no\n          # ~/.codex/config.toml and codex would only emit a confusing\n          # connection-refused against the dead local proxy. Detect that absence up\n          # front and fail-closed with a clear message instead.\n          if [ ! -f \"${CODEX_HOME:-$HOME/.codex}/config.toml\" ]; then\n            echo \"::error title=Moon Bridge unavailable::codex DeepSeek bridge setup did not complete (no config.toml) — codex (required at L3) cannot run; fail-closing to 'not mergeable (codex unavailable)'.\"\n            echo \"codex_exit=1\" >> \"$GITHUB_OUTPUT\"\n          else\n            uv run python .github/scripts/codex_review.py \\\n              --review-pack review-pack \\\n              --artifacts jobs/integration-final \\\n              --deterministic-verdict \"$DETERMINISTIC_VERDICT\" \\\n              --findings-out review-pack/codex_findings.json \\\n              --codex-out review-pack/codex_output.txt\n            rc=$?\n            echo \"codex_exit=$rc\" >> \"$GITHUB_OUTPUT\"\n          fi\n          # The final verdict was written to GITHUB_OUTPUT by codex_review.py\n          # (final_verdict / codex_verdict). codex_exit != 0 => fail-closed.\n\n      - name: Compute the final published verdict\n        id: final\n        if: always()\n        env:\n          CODEX_REQUIRED: ${{ github.event.inputs.codex_review != 'false' }}\n          DET: ${{ steps.pack.outputs.deterministic_verdict }}\n          # The fail-closed trigger is an empty CODEX_FINAL (codex produced no\n          # verdict). codex_exit is intentionally not consulted: codex_review.py\n          # returns non-zero for a legitimately-computed \"not mergeable\" too, so\n          # gating on it would mislabel a real verdict as \"codex unavailable\".\n          CODEX_FINAL: ${{ steps.codex.outputs.final_verdict }}\n        run: |\n          set -euo pipefail\n          if [ \"$CODEX_REQUIRED\" = \"false\" ]; then\n            FINAL=\"$DET\"\n          elif [ -n \"$CODEX_FINAL\" ]; then\n            FINAL=\"$CODEX_FINAL\"\n          else\n            # Codex was required but produced no verdict — fail closed.\n            FINAL=\"not mergeable (codex unavailable)\"\n          fi\n          echo \"final_verdict=$FINAL\" >> \"$GITHUB_OUTPUT\"\n          case \"$FINAL\" in\n            \"mergeable\"|\"mergeable with quarantines\") echo \"conclusion=success\" >> \"$GITHUB_OUTPUT\" ;;\n            *) echo \"conclusion=failure\" >> \"$GITHUB_OUTPUT\" ;;\n          esac\n          echo \"FINAL VERDICT: $FINAL\"\n\n      - name: Upsert final-review PR comment\n        if: always()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          REPO: ${{ github.repository }}\n          PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n          MARKER: \"<!-- integration-final-review -->\"\n        run: |\n          set -euo pipefail\n          BODY_FILE=\"$RUNNER_TEMP/final-comment.md\"\n          {\n            printf '%s\\n\\n' \"$MARKER\"\n            printf '## Integration final review\\n\\n**Final verdict: %s**\\n\\n' \"$FINAL_VERDICT\"\n            cat review-pack/verdict.md\n          } > \"$BODY_FILE\"\n          EXISTING_ID=$(gh api \"repos/${REPO}/issues/${PR_NUMBER}/comments\" --paginate \\\n            --jq \".[] | select(.body | contains(\\\"${MARKER}\\\")) | .id\" | head -1 || echo \"\")\n          if [ -n \"$EXISTING_ID\" ]; then\n            gh api -X PATCH \"repos/${REPO}/issues/comments/${EXISTING_ID}\" -F body=@\"$BODY_FILE\" >/dev/null\n          else\n            gh api -X POST \"repos/${REPO}/issues/${PR_NUMBER}/comments\" -F body=@\"$BODY_FILE\" >/dev/null\n          fi\n\n      - name: Publish integration-final-review check-run\n        if: always()\n        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1\n        env:\n          HEAD_SHA: ${{ needs.plan.outputs.head_sha }}\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        with:\n          script: |\n            const fs = require('fs');\n            const headSha = process.env.HEAD_SHA;\n            const conclusion = process.env.CONCLUSION || 'failure';\n            const verdict = process.env.FINAL_VERDICT || 'not mergeable';\n            let summary = '';\n            try { summary = fs.readFileSync('review-pack/verdict.md', 'utf8'); }\n            catch (e) { summary = 'No review pack produced.'; }\n            if (summary.length > 60000) summary = summary.slice(0, 60000) + '\\n\\n…(truncated)';\n            await github.rest.checks.create({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              name: 'integration-final-review',\n              head_sha: headSha,\n              status: 'completed',\n              conclusion: conclusion,\n              output: {\n                title: `integration-final-review: ${verdict}`,\n                summary: summary,\n              },\n            });\n\n      - name: Upload review pack\n        if: always()\n        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0\n        with:\n          name: integration-final-review-pack\n          path: review-pack\n          if-no-files-found: ignore\n          retention-days: 30\n\n      - name: Enforce the final verdict (gate)\n        if: always()\n        env:\n          CONCLUSION: ${{ steps.final.outputs.conclusion }}\n          FINAL_VERDICT: ${{ steps.final.outputs.final_verdict }}\n        run: |\n          if [ \"$CONCLUSION\" != \"success\" ]; then\n            echo \"::error::integration final review verdict: $FINAL_VERDICT\"\n            exit 1\n          fi\n          echo \"integration final review verdict: $FINAL_VERDICT\"\n"}}, "src/benchflow/agents/providers.py": {"cf656179f3bc30b2887b744cf71b69b5e5ffa5ff": {"t": 1785745456.140041, "n": 21375, "h": "5367df33d3dbd95f", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``model_prefixes``   Bare-model-name family tokens this provider owns\n                       (e.g. ``[\"deepseek\"]``), used by\n                       ``find_provider_for_bare_model()`` to route\n                       prefix-stripped ids. Tokens must be lowercase and\n                       unique across providers (longest token wins).\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    # Fallback value per url_params placeholder when its env var is unset, so a\n    # provider can ship a usable default endpoint while still honoring an\n    # explicit override (the env var wins). Empty => the env var is required\n    # (historical behavior).\n    url_param_defaults: dict[str, str] = field(default_factory=dict)\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Bare-model-name family tokens this provider owns (e.g. [\"deepseek\"],\n    # [\"qwen\"]). Used by find_provider_for_bare_model() to route a *prefix-less*\n    # model id (after strip_provider_prefix) — e.g. \"deepseek-v4-flash\" or\n    # \"qwen3.6-max\" — back to its provider when no \"provider/\" prefix is present.\n    # A token matches when the model id equals it or continues with a\n    # non-letter (version digit / \"-\" / \".\"), so \"glm\" matches \"glm-4.6\" and\n    # \"glm5\" but not \"glmnext\". Longest token wins across providers.\n    model_prefixes: list[str] = field(default_factory=list)\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # GitHub Models exposes an OpenAI-compatible endpoint that GitHub Actions\n    # can call with the workflow-scoped GITHUB_TOKEN and `models: read`.\n    \"github-models\": ProviderConfig(\n        name=\"github-models\",\n        base_url=\"https://models.github.ai/inference\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GITHUB_TOKEN\",\n    ),\n    # OpenRouter exposes hosted models through an OpenAI-compatible endpoint.\n    # Keep qwen-dashscope as the owner for bare qwen* models; callers choose\n    # OpenRouter explicitly with openrouter/<provider>/<model>.\n    \"openrouter\": ProviderConfig(\n        name=\"openrouter\",\n        base_url=\"https://openrouter.ai/api/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENROUTER_API_KEY\",\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n        model_prefixes=[\"kimi\", \"moonshot\"],\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n        model_prefixes=[\"minimax\"],\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n        model_prefixes=[\"qwen\"],\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        model_prefixes=[\"glm\"],\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n        # deepseek-v4-pro is served at deepseek's public OpenAI-compatible\n        # endpoint; default to it so bare ids resolve without DEEPSEEK_BASE_URL\n        # (matches the shim default), while DEEPSEEK_BASE_URL still overrides.\n        url_param_defaults={\"base_url\": \"https://api.deepseek.com/v1\"},\n        model_prefixes=[\"deepseek\"],\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n        model_prefixes=[\"xiaomi\", \"mimo\"],\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-lite\"],\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-pro\"],\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n        model_prefixes=[\"hunyuan\"],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef _bare_model_matches_token(model: str, token: str) -> bool:\n    \"\"\"True if a bare model id belongs to the family named by *token*.\n\n    Matches when the (lower-cased) model id equals the token, or starts with\n    it and the next character is *not* a letter — i.e. a version digit, \"-\",\n    or \".\" continues the same family. This routes both hyphen-style ids\n    (``deepseek-v4-flash`` → ``deepseek``, ``glm-4.6`` → ``glm``) and\n    version-suffixed ids (``qwen3.6-max`` → ``qwen``), while refusing\n    different words (``glmnext`` does NOT match ``glm``).\n    \"\"\"\n    if model == token:\n        return True\n    if model.startswith(token):\n        return not model[len(token)].isalpha()\n    return False\n\n\ndef find_provider_for_bare_model(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Map a BARE (prefix-stripped) model id to a registered provider.\n\n    ``find_provider`` only matches an explicit ``provider/`` prefix, so once\n    ``strip_provider_prefix`` has run (ACP set_model, BENCHFLOW_PROVIDER_MODEL,\n    Harbor YAML) a bare id like ``deepseek-v4-flash`` no longer resolves and\n    callers wrongly default it to anthropic. This consults each provider's\n    declared ``model_prefixes`` (the registry owns that knowledge) so any\n    registered provider routes correctly without the prefix.\n\n    Resolution (deterministic):\n      1. Longest ``model_prefixes`` token match wins across all providers\n         (e.g. ``doubao-seed-2-pro`` beats a hypothetical ``doubao``).\n      2. If no token matches, an exact ``models[].id`` declared by exactly\n         one provider is used as a fallback (ambiguous ids are skipped).\n\n    Returns ``(provider_name, config)`` or ``None``. Inputs that still carry a\n    registered ``provider/`` prefix return ``None`` (use ``find_provider``).\n    \"\"\"\n    m = model.lower().strip()\n    if not m or find_provider(m) is not None:\n        return None\n\n    # 1. Longest model-family-token match.\n    best: tuple[int, str, ProviderConfig] | None = None\n    for name, cfg in PROVIDERS.items():\n        for token in cfg.model_prefixes:\n            t = token.lower()\n            if _bare_model_matches_token(m, t) and (best is None or len(t) > best[0]):\n                best = (len(t), name, cfg)\n    if best is not None:\n        return best[1], best[2]\n\n    # 2. Exact declared-model-id fallback (only when unambiguous).\n    matches = [\n        (name, cfg)\n        for name, cfg in PROVIDERS.items()\n        if any(str(meta.get(\"id\", \"\")).lower() == m for meta in cfg.models)\n    ]\n    if len(matches) == 1:\n        return matches[0]\n    return None\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var) or provider.url_param_defaults.get(placeholder)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "cc1bf369cac37b058330eaf22dde0d5f350f32cb": {"t": 1785749197.1261342, "n": 21375, "h": "5367df33d3dbd95f", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``model_prefixes``   Bare-model-name family tokens this provider owns\n                       (e.g. ``[\"deepseek\"]``), used by\n                       ``find_provider_for_bare_model()`` to route\n                       prefix-stripped ids. Tokens must be lowercase and\n                       unique across providers (longest token wins).\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    # Fallback value per url_params placeholder when its env var is unset, so a\n    # provider can ship a usable default endpoint while still honoring an\n    # explicit override (the env var wins). Empty => the env var is required\n    # (historical behavior).\n    url_param_defaults: dict[str, str] = field(default_factory=dict)\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Bare-model-name family tokens this provider owns (e.g. [\"deepseek\"],\n    # [\"qwen\"]). Used by find_provider_for_bare_model() to route a *prefix-less*\n    # model id (after strip_provider_prefix) — e.g. \"deepseek-v4-flash\" or\n    # \"qwen3.6-max\" — back to its provider when no \"provider/\" prefix is present.\n    # A token matches when the model id equals it or continues with a\n    # non-letter (version digit / \"-\" / \".\"), so \"glm\" matches \"glm-4.6\" and\n    # \"glm5\" but not \"glmnext\". Longest token wins across providers.\n    model_prefixes: list[str] = field(default_factory=list)\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # GitHub Models exposes an OpenAI-compatible endpoint that GitHub Actions\n    # can call with the workflow-scoped GITHUB_TOKEN and `models: read`.\n    \"github-models\": ProviderConfig(\n        name=\"github-models\",\n        base_url=\"https://models.github.ai/inference\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GITHUB_TOKEN\",\n    ),\n    # OpenRouter exposes hosted models through an OpenAI-compatible endpoint.\n    # Keep qwen-dashscope as the owner for bare qwen* models; callers choose\n    # OpenRouter explicitly with openrouter/<provider>/<model>.\n    \"openrouter\": ProviderConfig(\n        name=\"openrouter\",\n        base_url=\"https://openrouter.ai/api/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENROUTER_API_KEY\",\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n        model_prefixes=[\"kimi\", \"moonshot\"],\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n        model_prefixes=[\"minimax\"],\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n        model_prefixes=[\"qwen\"],\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        model_prefixes=[\"glm\"],\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n        # deepseek-v4-pro is served at deepseek's public OpenAI-compatible\n        # endpoint; default to it so bare ids resolve without DEEPSEEK_BASE_URL\n        # (matches the shim default), while DEEPSEEK_BASE_URL still overrides.\n        url_param_defaults={\"base_url\": \"https://api.deepseek.com/v1\"},\n        model_prefixes=[\"deepseek\"],\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n        model_prefixes=[\"xiaomi\", \"mimo\"],\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-lite\"],\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-pro\"],\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n        model_prefixes=[\"hunyuan\"],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef _bare_model_matches_token(model: str, token: str) -> bool:\n    \"\"\"True if a bare model id belongs to the family named by *token*.\n\n    Matches when the (lower-cased) model id equals the token, or starts with\n    it and the next character is *not* a letter — i.e. a version digit, \"-\",\n    or \".\" continues the same family. This routes both hyphen-style ids\n    (``deepseek-v4-flash`` → ``deepseek``, ``glm-4.6`` → ``glm``) and\n    version-suffixed ids (``qwen3.6-max`` → ``qwen``), while refusing\n    different words (``glmnext`` does NOT match ``glm``).\n    \"\"\"\n    if model == token:\n        return True\n    if model.startswith(token):\n        return not model[len(token)].isalpha()\n    return False\n\n\ndef find_provider_for_bare_model(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Map a BARE (prefix-stripped) model id to a registered provider.\n\n    ``find_provider`` only matches an explicit ``provider/`` prefix, so once\n    ``strip_provider_prefix`` has run (ACP set_model, BENCHFLOW_PROVIDER_MODEL,\n    Harbor YAML) a bare id like ``deepseek-v4-flash`` no longer resolves and\n    callers wrongly default it to anthropic. This consults each provider's\n    declared ``model_prefixes`` (the registry owns that knowledge) so any\n    registered provider routes correctly without the prefix.\n\n    Resolution (deterministic):\n      1. Longest ``model_prefixes`` token match wins across all providers\n         (e.g. ``doubao-seed-2-pro`` beats a hypothetical ``doubao``).\n      2. If no token matches, an exact ``models[].id`` declared by exactly\n         one provider is used as a fallback (ambiguous ids are skipped).\n\n    Returns ``(provider_name, config)`` or ``None``. Inputs that still carry a\n    registered ``provider/`` prefix return ``None`` (use ``find_provider``).\n    \"\"\"\n    m = model.lower().strip()\n    if not m or find_provider(m) is not None:\n        return None\n\n    # 1. Longest model-family-token match.\n    best: tuple[int, str, ProviderConfig] | None = None\n    for name, cfg in PROVIDERS.items():\n        for token in cfg.model_prefixes:\n            t = token.lower()\n            if _bare_model_matches_token(m, t) and (best is None or len(t) > best[0]):\n                best = (len(t), name, cfg)\n    if best is not None:\n        return best[1], best[2]\n\n    # 2. Exact declared-model-id fallback (only when unambiguous).\n    matches = [\n        (name, cfg)\n        for name, cfg in PROVIDERS.items()\n        if any(str(meta.get(\"id\", \"\")).lower() == m for meta in cfg.models)\n    ]\n    if len(matches) == 1:\n        return matches[0]\n    return None\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var) or provider.url_param_defaults.get(placeholder)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "8554d892ba4018f359a80d134be186fa9ae32320": {"t": 1785749198.2848125, "n": 20928, "h": "63724bf97e1a9161", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``model_prefixes``   Bare-model-name family tokens this provider owns\n                       (e.g. ``[\"deepseek\"]``), used by\n                       ``find_provider_for_bare_model()`` to route\n                       prefix-stripped ids. Tokens must be lowercase and\n                       unique across providers (longest token wins).\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    # Fallback value per url_params placeholder when its env var is unset, so a\n    # provider can ship a usable default endpoint while still honoring an\n    # explicit override (the env var wins). Empty => the env var is required\n    # (historical behavior).\n    url_param_defaults: dict[str, str] = field(default_factory=dict)\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Bare-model-name family tokens this provider owns (e.g. [\"deepseek\"],\n    # [\"qwen\"]). Used by find_provider_for_bare_model() to route a *prefix-less*\n    # model id (after strip_provider_prefix) — e.g. \"deepseek-v4-flash\" or\n    # \"qwen3.6-max\" — back to its provider when no \"provider/\" prefix is present.\n    # A token matches when the model id equals it or continues with a\n    # non-letter (version digit / \"-\" / \".\"), so \"glm\" matches \"glm-4.6\" and\n    # \"glm5\" but not \"glmnext\". Longest token wins across providers.\n    model_prefixes: list[str] = field(default_factory=list)\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # GitHub Models exposes an OpenAI-compatible endpoint that GitHub Actions\n    # can call with the workflow-scoped GITHUB_TOKEN and `models: read`.\n    \"github-models\": ProviderConfig(\n        name=\"github-models\",\n        base_url=\"https://models.github.ai/inference\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GITHUB_TOKEN\",\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n        model_prefixes=[\"kimi\", \"moonshot\"],\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n        model_prefixes=[\"minimax\"],\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n        model_prefixes=[\"qwen\"],\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        model_prefixes=[\"glm\"],\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n        # deepseek-v4-pro is served at deepseek's public OpenAI-compatible\n        # endpoint; default to it so bare ids resolve without DEEPSEEK_BASE_URL\n        # (matches the shim default), while DEEPSEEK_BASE_URL still overrides.\n        url_param_defaults={\"base_url\": \"https://api.deepseek.com/v1\"},\n        model_prefixes=[\"deepseek\"],\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n        model_prefixes=[\"xiaomi\", \"mimo\"],\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-lite\"],\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-pro\"],\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n        model_prefixes=[\"hunyuan\"],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef _bare_model_matches_token(model: str, token: str) -> bool:\n    \"\"\"True if a bare model id belongs to the family named by *token*.\n\n    Matches when the (lower-cased) model id equals the token, or starts with\n    it and the next character is *not* a letter — i.e. a version digit, \"-\",\n    or \".\" continues the same family. This routes both hyphen-style ids\n    (``deepseek-v4-flash`` → ``deepseek``, ``glm-4.6`` → ``glm``) and\n    version-suffixed ids (``qwen3.6-max`` → ``qwen``), while refusing\n    different words (``glmnext`` does NOT match ``glm``).\n    \"\"\"\n    if model == token:\n        return True\n    if model.startswith(token):\n        return not model[len(token)].isalpha()\n    return False\n\n\ndef find_provider_for_bare_model(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Map a BARE (prefix-stripped) model id to a registered provider.\n\n    ``find_provider`` only matches an explicit ``provider/`` prefix, so once\n    ``strip_provider_prefix`` has run (ACP set_model, BENCHFLOW_PROVIDER_MODEL,\n    Harbor YAML) a bare id like ``deepseek-v4-flash`` no longer resolves and\n    callers wrongly default it to anthropic. This consults each provider's\n    declared ``model_prefixes`` (the registry owns that knowledge) so any\n    registered provider routes correctly without the prefix.\n\n    Resolution (deterministic):\n      1. Longest ``model_prefixes`` token match wins across all providers\n         (e.g. ``doubao-seed-2-pro`` beats a hypothetical ``doubao``).\n      2. If no token matches, an exact ``models[].id`` declared by exactly\n         one provider is used as a fallback (ambiguous ids are skipped).\n\n    Returns ``(provider_name, config)`` or ``None``. Inputs that still carry a\n    registered ``provider/`` prefix return ``None`` (use ``find_provider``).\n    \"\"\"\n    m = model.lower().strip()\n    if not m or find_provider(m) is not None:\n        return None\n\n    # 1. Longest model-family-token match.\n    best: tuple[int, str, ProviderConfig] | None = None\n    for name, cfg in PROVIDERS.items():\n        for token in cfg.model_prefixes:\n            t = token.lower()\n            if _bare_model_matches_token(m, t) and (best is None or len(t) > best[0]):\n                best = (len(t), name, cfg)\n    if best is not None:\n        return best[1], best[2]\n\n    # 2. Exact declared-model-id fallback (only when unambiguous).\n    matches = [\n        (name, cfg)\n        for name, cfg in PROVIDERS.items()\n        if any(str(meta.get(\"id\", \"\")).lower() == m for meta in cfg.models)\n    ]\n    if len(matches) == 1:\n        return matches[0]\n    return None\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var) or provider.url_param_defaults.get(placeholder)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "66448e07a2950fa92cba19919cb37b367e5a9abf": {"t": 1785749200.1500793, "n": 20245, "h": "8ba99aaa6f961774", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``model_prefixes``   Bare-model-name family tokens this provider owns\n                       (e.g. ``[\"deepseek\"]``), used by\n                       ``find_provider_for_bare_model()`` to route\n                       prefix-stripped ids. Tokens must be lowercase and\n                       unique across providers (longest token wins).\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Bare-model-name family tokens this provider owns (e.g. [\"deepseek\"],\n    # [\"qwen\"]). Used by find_provider_for_bare_model() to route a *prefix-less*\n    # model id (after strip_provider_prefix) — e.g. \"deepseek-v4-flash\" or\n    # \"qwen3.6-max\" — back to its provider when no \"provider/\" prefix is present.\n    # A token matches when the model id equals it or continues with a\n    # non-letter (version digit / \"-\" / \".\"), so \"glm\" matches \"glm-4.6\" and\n    # \"glm5\" but not \"glmnext\". Longest token wins across providers.\n    model_prefixes: list[str] = field(default_factory=list)\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # GitHub Models exposes an OpenAI-compatible endpoint that GitHub Actions\n    # can call with the workflow-scoped GITHUB_TOKEN and `models: read`.\n    \"github-models\": ProviderConfig(\n        name=\"github-models\",\n        base_url=\"https://models.github.ai/inference\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GITHUB_TOKEN\",\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n        model_prefixes=[\"kimi\", \"moonshot\"],\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n        model_prefixes=[\"minimax\"],\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n        model_prefixes=[\"qwen\"],\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        model_prefixes=[\"glm\"],\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n        model_prefixes=[\"deepseek\"],\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n        model_prefixes=[\"xiaomi\", \"mimo\"],\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-lite\"],\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-pro\"],\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n        model_prefixes=[\"hunyuan\"],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef _bare_model_matches_token(model: str, token: str) -> bool:\n    \"\"\"True if a bare model id belongs to the family named by *token*.\n\n    Matches when the (lower-cased) model id equals the token, or starts with\n    it and the next character is *not* a letter — i.e. a version digit, \"-\",\n    or \".\" continues the same family. This routes both hyphen-style ids\n    (``deepseek-v4-flash`` → ``deepseek``, ``glm-4.6`` → ``glm``) and\n    version-suffixed ids (``qwen3.6-max`` → ``qwen``), while refusing\n    different words (``glmnext`` does NOT match ``glm``).\n    \"\"\"\n    if model == token:\n        return True\n    if model.startswith(token):\n        return not model[len(token)].isalpha()\n    return False\n\n\ndef find_provider_for_bare_model(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Map a BARE (prefix-stripped) model id to a registered provider.\n\n    ``find_provider`` only matches an explicit ``provider/`` prefix, so once\n    ``strip_provider_prefix`` has run (ACP set_model, BENCHFLOW_PROVIDER_MODEL,\n    Harbor YAML) a bare id like ``deepseek-v4-flash`` no longer resolves and\n    callers wrongly default it to anthropic. This consults each provider's\n    declared ``model_prefixes`` (the registry owns that knowledge) so any\n    registered provider routes correctly without the prefix.\n\n    Resolution (deterministic):\n      1. Longest ``model_prefixes`` token match wins across all providers\n         (e.g. ``doubao-seed-2-pro`` beats a hypothetical ``doubao``).\n      2. If no token matches, an exact ``models[].id`` declared by exactly\n         one provider is used as a fallback (ambiguous ids are skipped).\n\n    Returns ``(provider_name, config)`` or ``None``. Inputs that still carry a\n    registered ``provider/`` prefix return ``None`` (use ``find_provider``).\n    \"\"\"\n    m = model.lower().strip()\n    if not m or find_provider(m) is not None:\n        return None\n\n    # 1. Longest model-family-token match.\n    best: tuple[int, str, ProviderConfig] | None = None\n    for name, cfg in PROVIDERS.items():\n        for token in cfg.model_prefixes:\n            t = token.lower()\n            if _bare_model_matches_token(m, t) and (best is None or len(t) > best[0]):\n                best = (len(t), name, cfg)\n    if best is not None:\n        return best[1], best[2]\n\n    # 2. Exact declared-model-id fallback (only when unambiguous).\n    matches = [\n        (name, cfg)\n        for name, cfg in PROVIDERS.items()\n        if any(str(meta.get(\"id\", \"\")).lower() == m for meta in cfg.models)\n    ]\n    if len(matches) == 1:\n        return matches[0]\n    return None\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "961fc62aee88a343038f7af7a16b4421bb93564e": {"t": 1785749202.3368907, "n": 19860, "h": "734e760ee67ad9a8", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``model_prefixes``   Bare-model-name family tokens this provider owns\n                       (e.g. ``[\"deepseek\"]``), used by\n                       ``find_provider_for_bare_model()`` to route\n                       prefix-stripped ids. Tokens must be lowercase and\n                       unique across providers (longest token wins).\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Bare-model-name family tokens this provider owns (e.g. [\"deepseek\"],\n    # [\"qwen\"]). Used by find_provider_for_bare_model() to route a *prefix-less*\n    # model id (after strip_provider_prefix) — e.g. \"deepseek-v4-flash\" or\n    # \"qwen3.6-max\" — back to its provider when no \"provider/\" prefix is present.\n    # A token matches when the model id equals it or continues with a\n    # non-letter (version digit / \"-\" / \".\"), so \"glm\" matches \"glm-4.6\" and\n    # \"glm5\" but not \"glmnext\". Longest token wins across providers.\n    model_prefixes: list[str] = field(default_factory=list)\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n        model_prefixes=[\"kimi\", \"moonshot\"],\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n        model_prefixes=[\"minimax\"],\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n        model_prefixes=[\"qwen\"],\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        model_prefixes=[\"glm\"],\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n        model_prefixes=[\"deepseek\"],\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n        model_prefixes=[\"xiaomi\", \"mimo\"],\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-lite\"],\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-pro\"],\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n        model_prefixes=[\"hunyuan\"],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef _bare_model_matches_token(model: str, token: str) -> bool:\n    \"\"\"True if a bare model id belongs to the family named by *token*.\n\n    Matches when the (lower-cased) model id equals the token, or starts with\n    it and the next character is *not* a letter — i.e. a version digit, \"-\",\n    or \".\" continues the same family. This routes both hyphen-style ids\n    (``deepseek-v4-flash`` → ``deepseek``, ``glm-4.6`` → ``glm``) and\n    version-suffixed ids (``qwen3.6-max`` → ``qwen``), while refusing\n    different words (``glmnext`` does NOT match ``glm``).\n    \"\"\"\n    if model == token:\n        return True\n    if model.startswith(token):\n        return not model[len(token)].isalpha()\n    return False\n\n\ndef find_provider_for_bare_model(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Map a BARE (prefix-stripped) model id to a registered provider.\n\n    ``find_provider`` only matches an explicit ``provider/`` prefix, so once\n    ``strip_provider_prefix`` has run (ACP set_model, BENCHFLOW_PROVIDER_MODEL,\n    Harbor YAML) a bare id like ``deepseek-v4-flash`` no longer resolves and\n    callers wrongly default it to anthropic. This consults each provider's\n    declared ``model_prefixes`` (the registry owns that knowledge) so any\n    registered provider routes correctly without the prefix.\n\n    Resolution (deterministic):\n      1. Longest ``model_prefixes`` token match wins across all providers\n         (e.g. ``doubao-seed-2-pro`` beats a hypothetical ``doubao``).\n      2. If no token matches, an exact ``models[].id`` declared by exactly\n         one provider is used as a fallback (ambiguous ids are skipped).\n\n    Returns ``(provider_name, config)`` or ``None``. Inputs that still carry a\n    registered ``provider/`` prefix return ``None`` (use ``find_provider``).\n    \"\"\"\n    m = model.lower().strip()\n    if not m or find_provider(m) is not None:\n        return None\n\n    # 1. Longest model-family-token match.\n    best: tuple[int, str, ProviderConfig] | None = None\n    for name, cfg in PROVIDERS.items():\n        for token in cfg.model_prefixes:\n            t = token.lower()\n            if _bare_model_matches_token(m, t) and (best is None or len(t) > best[0]):\n                best = (len(t), name, cfg)\n    if best is not None:\n        return best[1], best[2]\n\n    # 2. Exact declared-model-id fallback (only when unambiguous).\n    matches = [\n        (name, cfg)\n        for name, cfg in PROVIDERS.items()\n        if any(str(meta.get(\"id\", \"\")).lower() == m for meta in cfg.models)\n    ]\n    if len(matches) == 1:\n        return matches[0]\n    return None\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "3e7301523683ea490845369a697b16c0f5adf5e8": {"t": 1785749203.4532685, "n": 19523, "h": "ac76e447bb3e2eb3", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Bare-model-name family tokens this provider owns (e.g. [\"deepseek\"],\n    # [\"qwen\"]). Used by find_provider_for_bare_model() to route a *prefix-less*\n    # model id (after strip_provider_prefix) — e.g. \"deepseek-v4-flash\" or\n    # \"qwen3.6-max\" — back to its provider when no \"provider/\" prefix is present.\n    # A token matches when the model id equals it or continues with a\n    # non-letter (version digit / \"-\" / \".\"), so \"glm\" matches \"glm-4.6\" and\n    # \"glm5\" but not \"glmnext\". Longest token wins across providers.\n    model_prefixes: list[str] = field(default_factory=list)\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n        model_prefixes=[\"kimi\", \"moonshot\"],\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n        model_prefixes=[\"minimax\"],\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n        model_prefixes=[\"qwen\"],\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        model_prefixes=[\"glm\"],\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n        model_prefixes=[\"deepseek\"],\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n        model_prefixes=[\"xiaomi\", \"mimo\"],\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-lite\"],\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-pro\"],\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n        model_prefixes=[\"hunyuan\"],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef _bare_model_matches_token(model: str, token: str) -> bool:\n    \"\"\"True if a bare model id belongs to the family named by *token*.\n\n    Matches when the (lower-cased) model id equals the token, or starts with\n    it and the next character is *not* a letter — i.e. a version digit, \"-\",\n    or \".\" continues the same family. This routes both hyphen-style ids\n    (``deepseek-v4-flash`` → ``deepseek``, ``glm-4.6`` → ``glm``) and\n    version-suffixed ids (``qwen3.6-max`` → ``qwen``), while refusing\n    different words (``glmnext`` does NOT match ``glm``).\n    \"\"\"\n    if model == token:\n        return True\n    if model.startswith(token):\n        return not model[len(token)].isalpha()\n    return False\n\n\ndef find_provider_for_bare_model(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Map a BARE (prefix-stripped) model id to a registered provider.\n\n    ``find_provider`` only matches an explicit ``provider/`` prefix, so once\n    ``strip_provider_prefix`` has run (ACP set_model, BENCHFLOW_PROVIDER_MODEL,\n    Harbor YAML) a bare id like ``deepseek-v4-flash`` no longer resolves and\n    callers wrongly default it to anthropic. This consults each provider's\n    declared ``model_prefixes`` (the registry owns that knowledge) so any\n    registered provider routes correctly without the prefix.\n\n    Resolution (deterministic):\n      1. Longest ``model_prefixes`` token match wins across all providers\n         (e.g. ``doubao-seed-2-pro`` beats a hypothetical ``doubao``).\n      2. If no token matches, an exact ``models[].id`` declared by exactly\n         one provider is used as a fallback (ambiguous ids are skipped).\n\n    Returns ``(provider_name, config)`` or ``None``. Inputs that still carry a\n    registered ``provider/`` prefix return ``None`` (use ``find_provider``).\n    \"\"\"\n    m = model.lower().strip()\n    if not m or find_provider(m) is not None:\n        return None\n\n    # 1. Longest model-family-token match.\n    best: tuple[int, str, ProviderConfig] | None = None\n    for name, cfg in PROVIDERS.items():\n        for token in cfg.model_prefixes:\n            t = token.lower()\n            if _bare_model_matches_token(m, t) and (best is None or len(t) > best[0]):\n                best = (len(t), name, cfg)\n    if best is not None:\n        return best[1], best[2]\n\n    # 2. Exact declared-model-id fallback (only when unambiguous).\n    matches = [\n        (name, cfg)\n        for name, cfg in PROVIDERS.items()\n        if any(str(meta.get(\"id\", \"\")).lower() == m for meta in cfg.models)\n    ]\n    if len(matches) == 1:\n        return matches[0]\n    return None\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "415e8ae093e9e0bf7ba29d43ac71abe34a39e5ee": {"t": 1785749204.5764077, "n": 15988, "h": "37a83a1183041e4b", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "240670b4fc6092c64b354a0c497a7f928fd0f413": {"t": 1785749205.901915, "n": 16000, "h": "0d8117c5f9b5bf7e", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # local Bedrock proxy supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "cbb802353798e267aa1809ffd51e5209768fbdf6": {"t": 1785749207.4329123, "n": 15079, "h": "8dfb9fe4e0f5777e", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # local Bedrock proxy supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "37aab8ae3e8d6dda467a3967d6d466c76bdaa3b8": {"t": 1785749208.5747664, "n": 12127, "h": "aa230ee6dc5300b7", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # local Bedrock proxy supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "789fbc0c3a633542d87c7b8b63089ea1634bebc2": {"t": 1785749209.7086725, "n": 11987, "h": "8e4b0ff0e5f954b1", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        api_protocol=\"openai-responses\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-completions\": \"https://{resource}.openai.azure.com/openai/v1\",\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # local Bedrock proxy supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "585ba727d623c62ba63eac0963e179a172779903": {"t": 1785749210.9514596, "n": 11947, "h": "0fc2bb93f4763c16", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        api_protocol=\"openai-responses\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_AI_FOUNDRY_API_KEY\",\n        url_params={\n            \"resource\": \"AZURE_AI_FOUNDRY_RESOURCE\",\n        },\n        endpoints={\n            \"openai-completions\": \"https://{resource}.openai.azure.com/openai/v1\",\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_AI_FOUNDRY_API_KEY\",\n        url_params={\n            \"resource\": \"AZURE_AI_FOUNDRY_RESOURCE\",\n        },\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # local Bedrock proxy supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "3e8b06cd9e8bf8e131fe1d4fc85a97caaf1f1711": {"t": 1785749213.135683, "n": 10827, "h": "28857737cdbf2fce", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # local Bedrock proxy supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "a585b147850f9e3919069499cfd46ba0e1f96562": {"t": 1785749214.6799545, "n": 10820, "h": "9e1e720a3b22dd6d", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # local Bedrock proxy supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}, "b7ffa70735b0b29f13ff90a943b1ddfbe644eff1": {"t": 1785749216.13501, "n": 10339, "h": "25d04a44cab430b3", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves both\n                       openai-completions and anthropic-messages). Picked\n                       at runtime based on the agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-completions\" | \"openai-responses\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "c813ba29400101748ebfb0b114fee77cd84940df": {"t": 1785749217.2251675, "n": 10228, "h": "370e924563e30a14", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\" or \"openai-completions\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves both\n                       openai-completions and anthropic-messages). Picked\n                       at runtime based on the agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: (\n        str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    )\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "9fa6a5ab9bc6e91c771e338bfe9e919abaa96ff2": {"t": 1785749218.3572307, "n": 10246, "h": "8644605453920782", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\" or \"openai-completions\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"none\": no auth (e.g. local vllm).\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves both\n                       openai-completions and anthropic-messages). Picked\n                       at runtime based on the agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: (\n        str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    )\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "83111d1266daaae6ce87c61a7db3e1a1b5070534": {"t": 1785749219.4852293, "n": 10172, "h": "09b4a16a2c9b3074", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\" or \"openai-completions\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"none\": no auth (e.g. local vllm).\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves both\n                       openai-completions and anthropic-messages). Picked\n                       at runtime based on the agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: (\n        str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    )\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"none\",\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "0e7909c88be22d949c2f21607450cff19594c270": {"t": 1785749221.0018578, "n": 9810, "h": "74606e9fb334882d", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\" or \"openai-completions\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"none\": no auth (e.g. local vllm).\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves both\n                       openai-completions and anthropic-messages). Picked\n                       at runtime based on the agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: (\n        str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    )\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"none\",\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "52254810bae890d228d421830a03da0af4c564f6": {"t": 1785749222.3130667, "n": 9796, "h": "db1de86a5f77eb50", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\" or \"openai-completions\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"none\": no auth (e.g. local vllm).\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves both\n                       openai-completions and anthropic-messages). Picked\n                       at runtime based on the agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: (\n        str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    )\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"none\",\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip the provider prefix from a model ID.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"  (no prefix = unchanged)\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        prefix = f\"{result[0]}/\"\n        return model[len(prefix) :]\n    # Not a known provider — still strip unknown prefix if present\n    if \"/\" in model:\n        return model.split(\"/\", 1)[1]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "1471947c4a1e377aa8c3c1f2147aea3bb1480416": {"t": 1785749224.4317362, "n": 7594, "h": "52aebf64ebdbbc0f", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so is_vertex_model() and infer_env_key_for_model()\nhave a single source of truth.\n\nNative providers have empty models lists — agents know them natively.\nCustom providers include model metadata so agent shims (e.g. openclaw)\ncan write the config files they need.\n\nAdding a new provider = one entry in PROVIDERS. No new functions needed.\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: (\n        str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    )\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"none\",\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip the provider prefix from a model ID.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"  (no prefix = unchanged)\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        prefix = f\"{result[0]}/\"\n        return model[len(prefix) :]\n    # Not a known provider — still strip unknown prefix if present\n    if \"/\" in model:\n        return model.split(\"/\", 1)[1]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "01ee396f5b0cb5a6d59ee7c95dc432ecb422c5c8": {"t": 1785749226.0030982, "n": 7439, "h": "1000775620b41316", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so is_vertex_model() and infer_env_key_for_model()\nhave a single source of truth.\n\nNative providers have empty models lists — agents know them natively.\nCustom providers include model metadata so agent shims (e.g. openclaw)\ncan write the config files they need.\n\nAdding a new provider = one entry in PROVIDERS. No new functions needed.\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: str  # primary endpoint; may contain {placeholders} expanded via url_params\n    api_protocol: str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n        credential_files=[{\n            \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n            \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n            \"post_env\": {\n                \"GOOGLE_APPLICATION_CREDENTIALS\":\n                    \"{home}/.config/gcloud/application_default_credentials.json\",\n            },\n        }],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n        credential_files=[{\n            \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n            \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n            \"post_env\": {\n                \"GOOGLE_APPLICATION_CREDENTIALS\":\n                    \"{home}/.config/gcloud/application_default_credentials.json\",\n            },\n        }],\n    ),\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --ae BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"none\",\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip the provider prefix from a model ID.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"  (no prefix = unchanged)\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        prefix = f\"{result[0]}/\"\n        return model[len(prefix):]\n    # Not a known provider — still strip unknown prefix if present\n    if \"/\" in model:\n        return model.split(\"/\", 1)[1]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "8996e1b13dbb39f4fabe4df9224b0506961a252a": {"t": 1785749227.0863998, "n": 7153, "h": "2571bb7154bf36ca", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so is_vertex_model() and infer_env_key_for_model()\nhave a single source of truth.\n\nNative providers have empty models lists — agents know them natively.\nCustom providers include model metadata so agent shims (e.g. openclaw)\ncan write the config files they need.\n\nAdding a new provider = one entry in PROVIDERS. No new functions needed.\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: str  # primary endpoint; may contain {placeholders} expanded via url_params\n    api_protocol: str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n        credential_files=[{\n            \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n            \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n            \"post_env\": {\n                \"GOOGLE_APPLICATION_CREDENTIALS\":\n                    \"{home}/.config/gcloud/application_default_credentials.json\",\n            },\n        }],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n        credential_files=[{\n            \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n            \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n            \"post_env\": {\n                \"GOOGLE_APPLICATION_CREDENTIALS\":\n                    \"{home}/.config/gcloud/application_default_credentials.json\",\n            },\n        }],\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip the provider prefix from a model ID.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"  (no prefix = unchanged)\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        prefix = f\"{result[0]}/\"\n        return model[len(prefix):]\n    # Not a known provider — still strip unknown prefix if present\n    if \"/\" in model:\n        return model.split(\"/\", 1)[1]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "059fb8bc72b84c8c3d1a2dfa4721afb48c856f2d": {"t": 1785749228.1754038, "n": 6237, "h": "25025893126c0a70", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so is_vertex_model() and infer_env_key_for_model()\nhave a single source of truth.\n\nNative providers have empty models lists — agents know them natively.\nCustom providers include model metadata so agent shims (e.g. openclaw)\ncan write the config files they need.\n\nAdding a new provider = one entry in PROVIDERS. No new functions needed.\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: str  # primary endpoint; may contain {placeholders} expanded via url_params\n    api_protocol: str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip the provider prefix from a model ID.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"  (no prefix = unchanged)\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        prefix = f\"{result[0]}/\"\n        return model[len(prefix):]\n    # Not a known provider — still strip unknown prefix if present\n    if \"/\" in model:\n        return model.split(\"/\", 1)[1]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "ad8f38fa1219d186e29f053d9707729bef458a92": {"t": 1785749229.4853327, "n": 5908, "h": "5dcf39f6c8dad260", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so is_vertex_model() and infer_env_key_for_model()\nhave a single source of truth.\n\nNative providers have empty models lists — agents know them natively.\nCustom providers include model metadata so agent shims (e.g. openclaw)\ncan write the config files they need.\n\nAdding a new provider = one entry in PROVIDERS. No new functions needed.\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: str  # primary endpoint; may contain {placeholders} expanded via url_params\n    api_protocol: str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip the provider prefix from a model ID.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"  (no prefix = unchanged)\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        prefix = f\"{result[0]}/\"\n        return model[len(prefix):]\n    # Not a known provider — still strip unknown prefix if present\n    if \"/\" in model:\n        return model.split(\"/\", 1)[1]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "4e39918ce47f3f2adc91421ea6a7ed6321c108e1": {"t": 1785749230.5733597, "n": 6631, "h": "1bd3a838988d99f9", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so is_vertex_model() and infer_env_key_for_model()\nhave a single source of truth.\n\nNative providers have empty models lists — agents know them natively.\nCustom providers include model metadata so agent shims (e.g. openclaw)\ncan write the config files they need.\n\nAdding a new provider = one entry in PROVIDERS. No new functions needed.\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: str  # primary endpoint; may contain {placeholders} expanded via url_params\n    api_protocol: str  # protocol for base_url: \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"vertex-zai\": ProviderConfig(\n        name=\"vertex-zai\",\n        base_url=(\n            \"https://aiplatform.googleapis.com/v1/projects/\"\n            \"{project_id}/locations/global/endpoints/openapi\"\n        ),\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\"},\n        models=[\n            {\n                \"id\": \"zai-org/glm-5-maas\",\n                \"name\": \"GLM-5 (Vertex AI)\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. vertex-zai/ vs vertex/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip the provider prefix from a model ID.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"  (no prefix = unchanged)\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        prefix = f\"{result[0]}/\"\n        return model[len(prefix):]\n    # Not a known provider — still strip unknown prefix if present\n    if \"/\" in model:\n        return model.split(\"/\", 1)[1]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "cb4cad20d496d8669547b4dd43c7eee9add89221": {"t": 1785749231.638874, "n": 5143, "h": "3a11dab1e09d5ebc", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so is_vertex_model() and infer_env_key_for_model()\nhave a single source of truth.\n\nNative providers have empty models lists — agents know them natively.\nCustom providers include model metadata so agent shims (e.g. openclaw)\ncan write the config files they need.\n\nAdding a new provider = one entry in PROVIDERS. No new functions needed.\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: str  # may contain {placeholders} expanded via url_params\n    api_protocol: str  # \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\", \"location\": \"GOOGLE_CLOUD_LOCATION\"},\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"vertex-zai\": ProviderConfig(\n        name=\"vertex-zai\",\n        base_url=(\n            \"https://aiplatform.googleapis.com/v1/projects/\"\n            \"{project_id}/locations/global/endpoints/openapi\"\n        ),\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\"project_id\": \"GOOGLE_CLOUD_PROJECT\"},\n        models=[\n            {\n                \"id\": \"zai-org/glm-5-maas\",\n                \"name\": \"GLM-5 (Vertex AI)\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. vertex-zai/ vs vertex/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef resolve_base_url(provider: ProviderConfig, env: dict[str, str]) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    if not provider.url_params:\n        return provider.base_url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return provider.base_url.format_map(replacements)\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based providers and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type == \"adc\":\n        return None\n    return cfg.auth_env\n"}, "2631b1d47436fbaa6017956c2957e7d4147b3cae": {"t": 1786068481.7071068, "n": 21375, "h": "5367df33d3dbd95f", "c": "\"\"\"LLM provider registry.\n\nEvery provider that benchflow routes models through lives here — both\n\"custom\" providers (like zai/) that need explicit endpoint config, and\n\"native\" providers (like google-vertex/) that agents already support\nbut we still register so ``is_vertex_model()`` and\n``infer_env_key_for_model()`` have a single source of truth.\n\nAdding a new provider is a registry-only change: append one entry to\n``PROVIDERS`` below. No new functions, no shim edits, no SDK edits.\n``tests/test_registry_invariants.py`` runs contract checks against every\nentry — read it for the executable schema.\n\nRequired fields\n---------------\n- ``name``           Must equal the dict key.\n- ``base_url``       Primary endpoint URL. May contain ``{placeholder}``\n                     tokens that get expanded from env vars via\n                     ``url_params``. Empty string is allowed for\n                     \"user-supplied at runtime\" providers (e.g. ``vllm``).\n- ``api_protocol``   \"anthropic-messages\", \"openai-completions\", or\n                     \"openai-responses\" — the\n                     wire protocol the primary ``base_url`` speaks.\n- ``auth_type``      \"api_key\" | \"adc\" | \"aws\" | \"none\".\n                     - \"api_key\": ``auth_env`` **must** be set.\n                     - \"adc\": Application Default Credentials (GCP). The\n                       SDK writes the credential file from\n                       ``credential_files`` and sets the corresponding env.\n                     - \"aws\": Bedrock API-key auth via\n                       ``AWS_BEARER_TOKEN_BEDROCK`` plus region.\n                     - \"none\": no auth.\n\nCommon optional fields\n----------------------\n- ``auth_env``         Env var holding the API key. Must be set iff\n                       ``auth_type == \"api_key\"``.\n- ``url_params``       ``{placeholder: ENV_VAR}`` — every placeholder in\n                       ``base_url`` (or any ``endpoints`` URL) must have an\n                       entry, and every entry must be referenced somewhere.\n- ``endpoints``        ``{api_protocol: url}`` for providers that expose\n                       multiple protocol surfaces (e.g. zai serves\n                       openai-responses, openai-completions, and\n                       anthropic-messages). Picked at runtime based on the\n                       agent's ``api_protocol``.\n- ``models``           Optional list of model metadata dicts (id, name,\n                       contextWindow, etc.) consumed by agent shims. ``id``\n                       is required and must be unique within the provider.\n- ``model_prefixes``   Bare-model-name family tokens this provider owns\n                       (e.g. ``[\"deepseek\"]``), used by\n                       ``find_provider_for_bare_model()`` to route\n                       prefix-stripped ids. Tokens must be lowercase and\n                       unique across providers (longest token wins).\n- ``credential_files`` List of dicts with ``\"path\"`` and ``\"env_source\"``\n                       (and optional ``\"post_env\"``) — used by ADC providers\n                       to write the credential blob into the container.\n\nLook at the existing entries below for worked examples:\n``zai`` (multi-endpoint, models metadata), ``google-vertex`` (ADC,\ncredential_files, url_params), ``vllm`` (user-supplied base_url).\n\"\"\"\n\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass ProviderConfig:\n    \"\"\"Configuration for a custom LLM provider.\"\"\"\n\n    name: str\n    base_url: (\n        str  # primary endpoint; may contain {placeholders} expanded via url_params\n    )\n    api_protocol: str  # protocol for base_url: \"openai-responses\" | \"openai-completions\" | \"anthropic-messages\"\n    auth_type: str  # \"api_key\" | \"adc\" | \"aws\" | \"none\"\n    auth_env: str | None = None  # env var holding the API key (None for ADC)\n    url_params: dict[str, str] = field(default_factory=dict)  # {placeholder: ENV_VAR}\n    # Fallback value per url_params placeholder when its env var is unset, so a\n    # provider can ship a usable default endpoint while still honoring an\n    # explicit override (the env var wins). Empty => the env var is required\n    # (historical behavior).\n    url_param_defaults: dict[str, str] = field(default_factory=dict)\n    models: list[dict] = field(default_factory=list)  # model metadata for agents\n    # Bare-model-name family tokens this provider owns (e.g. [\"deepseek\"],\n    # [\"qwen\"]). Used by find_provider_for_bare_model() to route a *prefix-less*\n    # model id (after strip_provider_prefix) — e.g. \"deepseek-v4-flash\" or\n    # \"qwen3.6-max\" — back to its provider when no \"provider/\" prefix is present.\n    # A token matches when the model id equals it or continues with a\n    # non-letter (version digit / \"-\" / \".\"), so \"glm\" matches \"glm-4.6\" and\n    # \"glm5\" but not \"glmnext\". Longest token wins across providers.\n    model_prefixes: list[str] = field(default_factory=list)\n    # Multi-protocol support: {protocol: base_url} for providers with multiple APIs.\n    # base_url + api_protocol is the primary; endpoints adds alternatives.\n    endpoints: dict[str, str] = field(default_factory=dict)\n    credential_files: list[dict] = field(default_factory=list)\n    # Files to write into container (e.g. GCP ADC).\n    # Each dict: {\"path\": str, \"env_source\": str, \"post_env\": {k: v} (optional)}\n\n    @property\n    def all_endpoints(self) -> dict[str, str]:\n        \"\"\"Merged view: endpoints dict with base_url/api_protocol as fallback.\"\"\"\n        merged = {self.api_protocol: self.base_url}\n        merged.update(self.endpoints)\n        return merged\n\n\n# ── Provider registry ──\n\nPROVIDERS: dict[str, ProviderConfig] = {\n    # ── Native Vertex AI providers (agents support these natively) ──\n    \"google-vertex\": ProviderConfig(\n        name=\"google-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    \"anthropic-vertex\": ProviderConfig(\n        name=\"anthropic-vertex\",\n        base_url=\"https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"adc\",\n        url_params={\n            \"project_id\": \"GOOGLE_CLOUD_PROJECT\",\n            \"location\": \"GOOGLE_CLOUD_LOCATION\",\n        },\n        credential_files=[\n            {\n                \"path\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                \"env_source\": \"GOOGLE_APPLICATION_CREDENTIALS_JSON\",\n                \"post_env\": {\n                    \"GOOGLE_APPLICATION_CREDENTIALS\": \"{home}/.config/gcloud/application_default_credentials.json\",\n                },\n            }\n        ],\n    ),\n    # ── Azure AI Foundry providers ──\n    #\n    # Foundry exposes different protocol surfaces. Keep those surfaces explicit\n    # in provider prefixes, while sharing one Azure resource/key env contract.\n    # AZURE_RESOURCE is normally derived from AZURE_API_ENDPOINT in env.py;\n    # users can also set it directly via --agent-env.\n    \"azure-foundry-openai\": ProviderConfig(\n        name=\"azure-foundry-openai\",\n        base_url=\"https://{resource}.openai.azure.com/openai/v1\",\n        # Use the broadest OpenAI-compatible default for agents that do not\n        # declare their own protocol (e.g. pi-acp). Responses-native agents\n        # still select the explicit openai-responses endpoint below.\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n        endpoints={\n            \"openai-responses\": \"https://{resource}.openai.azure.com/openai/v1\",\n        },\n    ),\n    \"azure-foundry-anthropic\": ProviderConfig(\n        name=\"azure-foundry-anthropic\",\n        base_url=\"https://{resource}.services.ai.azure.com/anthropic\",\n        api_protocol=\"anthropic-messages\",\n        auth_type=\"api_key\",\n        auth_env=\"AZURE_API_KEY\",\n        url_params={\"resource\": \"AZURE_RESOURCE\"},\n    ),\n    # ── OpenAI first-party API ──\n    \"openai\": ProviderConfig(\n        name=\"openai\",\n        base_url=\"https://api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.openai.com/v1\",\n            \"openai-responses\": \"https://api.openai.com/v1\",\n        },\n    ),\n    # OpenAI US data-residency endpoint. Same key, regional URL.\n    \"us-openai\": ProviderConfig(\n        name=\"us-openai\",\n        base_url=\"https://us.api.openai.com/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://us.api.openai.com/v1\",\n            \"openai-responses\": \"https://us.api.openai.com/v1\",\n        },\n    ),\n    # GitHub Models exposes an OpenAI-compatible endpoint that GitHub Actions\n    # can call with the workflow-scoped GITHUB_TOKEN and `models: read`.\n    \"github-models\": ProviderConfig(\n        name=\"github-models\",\n        base_url=\"https://models.github.ai/inference\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GITHUB_TOKEN\",\n    ),\n    # OpenRouter exposes hosted models through an OpenAI-compatible endpoint.\n    # Keep qwen-dashscope as the owner for bare qwen* models; callers choose\n    # OpenRouter explicitly with openrouter/<provider>/<model>.\n    \"openrouter\": ProviderConfig(\n        name=\"openrouter\",\n        base_url=\"https://openrouter.ai/api/v1\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENROUTER_API_KEY\",\n    ),\n    # TODO: add eu-openai (https://eu.api.openai.com/v1) when needed.\n    # ── OpenAI-compatible inference servers (user-supplied base_url) ──\n    \"vllm\": ProviderConfig(\n        name=\"vllm\",\n        base_url=\"\",  # user-supplied via --agent-env BENCHFLOW_PROVIDER_BASE_URL=...\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"OPENAI_API_KEY\",  # vLLM uses OpenAI-compatible auth\n    ),\n    \"litellm\": ProviderConfig(\n        name=\"litellm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"LITELLM_API_KEY\",\n        url_params={\"base_url\": \"LITELLM_BASE_URL\"},\n    ),\n    \"aws-bedrock\": ProviderConfig(\n        name=\"aws-bedrock\",\n        base_url=\"\",  # LiteLLM supplies the runtime URL later\n        api_protocol=\"openai-responses\",\n        auth_type=\"aws\",\n        endpoints={\n            \"anthropic-messages\": \"\",\n        },\n    ),\n    # ── Custom providers (need explicit endpoint config in agent shims) ──\n    \"zai\": ProviderConfig(\n        name=\"zai\",\n        base_url=\"https://api.z.ai/api/paas/v4\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"ZAI_API_KEY\",\n        endpoints={\n            \"openai-completions\": \"https://api.z.ai/api/paas/v4\",\n            \"openai-responses\": \"https://api.z.ai/api/paas/v4\",\n            \"anthropic-messages\": \"https://api.z.ai/api/anthropic\",\n        },\n        models=[\n            {\n                \"id\": \"glm-5\",\n                \"name\": \"GLM-5\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"kimi\": ProviderConfig(\n        name=\"kimi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"KIMI_API_KEY\",\n        url_params={\"base_url\": \"KIMI_BASE_URL\"},\n        model_prefixes=[\"kimi\", \"moonshot\"],\n    ),\n    \"minimax\": ProviderConfig(\n        name=\"minimax\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"MINIMAX_API_KEY\",\n        url_params={\"base_url\": \"MINIMAX_BASE_URL\"},\n        model_prefixes=[\"minimax\"],\n    ),\n    \"qwen-dashscope\": ProviderConfig(\n        name=\"qwen-dashscope\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"QWEN_API_KEY\",\n        url_params={\"base_url\": \"QWEN_BASE_URL\"},\n        model_prefixes=[\"qwen\"],\n    ),\n    \"glm\": ProviderConfig(\n        name=\"glm\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"GLM_API_KEY\",\n        url_params={\"base_url\": \"GLM_BASE_URL\"},\n        model_prefixes=[\"glm\"],\n        models=[\n            {\n                \"id\": \"glm-5.1\",\n                \"name\": \"GLM-5.1\",\n                \"reasoning\": True,\n                \"input\": [\"text\"],\n                \"cost\": {\"input\": 0, \"output\": 0, \"cacheRead\": 0, \"cacheWrite\": 0},\n                \"contextWindow\": 200000,\n                \"maxTokens\": 131072,\n            },\n        ],\n    ),\n    \"deepseek\": ProviderConfig(\n        name=\"deepseek\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DEEPSEEK_API_KEY\",\n        url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"},\n        # deepseek-v4-pro is served at deepseek's public OpenAI-compatible\n        # endpoint; default to it so bare ids resolve without DEEPSEEK_BASE_URL\n        # (matches the shim default), while DEEPSEEK_BASE_URL still overrides.\n        url_param_defaults={\"base_url\": \"https://api.deepseek.com/v1\"},\n        model_prefixes=[\"deepseek\"],\n    ),\n    \"xiaomi\": ProviderConfig(\n        name=\"xiaomi\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"XIAOMI_API_KEY\",\n        url_params={\"base_url\": \"XIAOMI_BASE_URL\"},\n        model_prefixes=[\"xiaomi\", \"mimo\"],\n    ),\n    \"doubao-seed-2-lite\": ProviderConfig(\n        name=\"doubao-seed-2-lite\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_LITE_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-lite\"],\n    ),\n    \"doubao-seed-2-pro\": ProviderConfig(\n        name=\"doubao-seed-2-pro\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"DOUBAO_SEED_2_PRO_API_KEY\",\n        url_params={\"base_url\": \"DOUBAO_VOLCES_BASE_URL\"},\n        model_prefixes=[\"doubao-seed-2-pro\"],\n    ),\n    \"hunyuan\": ProviderConfig(\n        name=\"hunyuan\",\n        base_url=\"{base_url}\",\n        api_protocol=\"openai-completions\",\n        auth_type=\"api_key\",\n        auth_env=\"HUNYUAN_API_KEY\",\n        url_params={\"base_url\": \"HUNYUAN_BASE_URL\"},\n        model_prefixes=[\"hunyuan\"],\n    ),\n}\n\n\ndef find_provider(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Find the custom provider for a model ID based on its prefix.\n\n    Returns (provider_name, config) or None if no custom provider matches.\n    Matches longest prefix first to handle nested prefixes (e.g. google-vertex/ vs google/).\n    \"\"\"\n    m = model.lower()\n    # Sort by prefix length descending so longer prefixes match first\n    candidates = []\n    for name, cfg in PROVIDERS.items():\n        prefix = f\"{name}/\"\n        if m.startswith(prefix):\n            candidates.append((len(prefix), name, cfg))\n    if not candidates:\n        return None\n    candidates.sort(reverse=True, key=lambda x: x[0])\n    _, name, cfg = candidates[0]\n    return name, cfg\n\n\ndef _bare_model_matches_token(model: str, token: str) -> bool:\n    \"\"\"True if a bare model id belongs to the family named by *token*.\n\n    Matches when the (lower-cased) model id equals the token, or starts with\n    it and the next character is *not* a letter — i.e. a version digit, \"-\",\n    or \".\" continues the same family. This routes both hyphen-style ids\n    (``deepseek-v4-flash`` → ``deepseek``, ``glm-4.6`` → ``glm``) and\n    version-suffixed ids (``qwen3.6-max`` → ``qwen``), while refusing\n    different words (``glmnext`` does NOT match ``glm``).\n    \"\"\"\n    if model == token:\n        return True\n    if model.startswith(token):\n        return not model[len(token)].isalpha()\n    return False\n\n\ndef find_provider_for_bare_model(model: str) -> tuple[str, ProviderConfig] | None:\n    \"\"\"Map a BARE (prefix-stripped) model id to a registered provider.\n\n    ``find_provider`` only matches an explicit ``provider/`` prefix, so once\n    ``strip_provider_prefix`` has run (ACP set_model, BENCHFLOW_PROVIDER_MODEL,\n    Harbor YAML) a bare id like ``deepseek-v4-flash`` no longer resolves and\n    callers wrongly default it to anthropic. This consults each provider's\n    declared ``model_prefixes`` (the registry owns that knowledge) so any\n    registered provider routes correctly without the prefix.\n\n    Resolution (deterministic):\n      1. Longest ``model_prefixes`` token match wins across all providers\n         (e.g. ``doubao-seed-2-pro`` beats a hypothetical ``doubao``).\n      2. If no token matches, an exact ``models[].id`` declared by exactly\n         one provider is used as a fallback (ambiguous ids are skipped).\n\n    Returns ``(provider_name, config)`` or ``None``. Inputs that still carry a\n    registered ``provider/`` prefix return ``None`` (use ``find_provider``).\n    \"\"\"\n    m = model.lower().strip()\n    if not m or find_provider(m) is not None:\n        return None\n\n    # 1. Longest model-family-token match.\n    best: tuple[int, str, ProviderConfig] | None = None\n    for name, cfg in PROVIDERS.items():\n        for token in cfg.model_prefixes:\n            t = token.lower()\n            if _bare_model_matches_token(m, t) and (best is None or len(t) > best[0]):\n                best = (len(t), name, cfg)\n    if best is not None:\n        return best[1], best[2]\n\n    # 2. Exact declared-model-id fallback (only when unambiguous).\n    matches = [\n        (name, cfg)\n        for name, cfg in PROVIDERS.items()\n        if any(str(meta.get(\"id\", \"\")).lower() == m for meta in cfg.models)\n    ]\n    if len(matches) == 1:\n        return matches[0]\n    return None\n\n\ndef resolve_base_url(\n    provider: ProviderConfig,\n    env: dict[str, str],\n    protocol: str | None = None,\n) -> str:\n    \"\"\"Expand {placeholders} in a provider's base_url using env vars.\n\n    If *protocol* is given and the provider has an ``endpoints`` entry for it,\n    that URL is used instead of the primary ``base_url``.\n\n    Raises KeyError if a required env var is missing.\n    \"\"\"\n    url = provider.base_url\n    if protocol and provider.endpoints.get(protocol):\n        url = provider.endpoints[protocol]\n    if not provider.url_params:\n        return url\n    replacements = {}\n    for placeholder, env_var in provider.url_params.items():\n        value = env.get(env_var) or provider.url_param_defaults.get(placeholder)\n        if not value:\n            raise KeyError(\n                f\"Provider {provider.name!r} requires {env_var} for \"\n                f\"{{{placeholder}}} in base_url, but it is not set.\"\n            )\n        replacements[placeholder] = value\n    return url.format_map(replacements)\n\n\ndef strip_provider_prefix(model: str) -> str:\n    \"\"\"Strip a *registered* provider prefix. Unregistered inputs pass through.\n\n    \"anthropic-vertex/claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n    \"zai/glm-5\" → \"glm-5\"\n    \"vllm/Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"  (HF org/model kept intact)\n    \"Qwen/Qwen3-Coder\" → \"Qwen/Qwen3-Coder\"       (no registered prefix → unchanged)\n    \"claude-sonnet-4-6\" → \"claude-sonnet-4-6\"\n\n    Single normalization point for downstream callers (ACP set_model,\n    BENCHFLOW_PROVIDER_MODEL env var, Harbor YAML parse). If a model ID\n    reaches an agent launcher still prefixed, fix the routing into this\n    function — do NOT strip again at the call site. See PRs #154 and #155\n    for the symptomatic-patch anti-pattern that caused the original bug.\n    \"\"\"\n    result = find_provider(model)\n    if result:\n        return model[len(result[0]) + 1 :]\n    return model\n\n\ndef resolve_auth_env(model: str) -> str | None:\n    \"\"\"Return the env var name needed for a model's provider, or None.\n\n    Returns None for ADC-based, AWS-auth providers, and unknown models.\n    \"\"\"\n    result = find_provider(model)\n    if result is None:\n        return None\n    _, cfg = result\n    if cfg.auth_type in {\"adc\", \"aws\", \"none\"}:\n        return None\n    return cfg.auth_env\n"}}, "tests/test_litellm_config.py": {"cf656179f3bc30b2887b744cf71b69b5e5ffa5ff": {"t": 1785746888.9178731, "n": 10822, "h": "feac7a868a0591b7", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_openai_route_honors_openhands_reasoning_effort_env():\n    \"\"\"Guards PR #911 against proxy-alias capability guessing.\"\"\"\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.6-sol\",\n        {\n            \"AZURE_API_KEY\": \"key\",\n            \"AZURE_RESOURCE\": \"benchflow\",\n            \"LLM_REASONING_EFFORT\": \"xhigh\",\n        },\n    )\n\n    assert route.litellm_params[\"reasoning_effort\"] == \"xhigh\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\n@pytest.mark.parametrize(\"model\", [\"gemini/gemini-2.5-flash\", \"gemini-2.5-flash\"])\ndef test_gemini_native_route_honors_explicit_base_url(model):\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        model,\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert route.provider_name == \"native\"\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_gemini_native_route_honors_generic_proxy_key():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/BENCHFLOW_PROVIDER_API_KEY\"\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_gemini_native_route_without_explicit_base_url_is_unchanged():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\"GEMINI_API_KEY\": \"sk-gemini\"},\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert \"api_base\" not in route.litellm_params\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_openrouter_route_uses_openai_compatible_endpoint():\n    route = resolve_litellm_route(\n        \"openrouter/qwen/qwen3.5-397b-a17b\",\n        {\"OPENROUTER_API_KEY\": \"sk-openrouter\"},\n    )\n\n    assert route.upstream_model == \"openai/qwen/qwen3.5-397b-a17b\"\n    assert route.litellm_params[\"api_base\"] == \"https://openrouter.ai/api/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/OPENROUTER_API_KEY\"\n    assert route.required_env == (\"OPENROUTER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_forces_chat_completions_for_anthropic_messages():\n    \"\"\"Streaming /v1/messages must bridge via /chat/completions so the\n    LiteLLM success callback fires and llm_trajectory.jsonl is written for\n    claude-agent-acp. LiteLLM's Responses-API streaming adapter (used for\n    openai/-prefixed upstreams such as the vllm provider) skips the success\n    callback; this flag opts out of it (#833).\n    \"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert (\n        config[\"litellm_settings\"][\"use_chat_completions_url_for_anthropic_messages\"]\n        is True\n    )\n\n\ndef test_litellm_exposes_anthropic_messages_chat_completions_flag():\n    \"\"\"Guard against a LiteLLM upgrade silently dropping the flag.\n\n    litellm_proxy_config sets use_chat_completions_url_for_anthropic_messages\n    via LiteLLM's generic litellm_settings -> setattr(litellm, ...) path, which\n    does NOT raise on an unknown key. If a future LiteLLM renames or removes\n    this attribute the fix would become a silent no-op and regress #833, so\n    assert the attribute still exists.\n    \"\"\"\n    import litellm\n\n    assert hasattr(litellm, \"use_chat_completions_url_for_anthropic_messages\")\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n\n\ndef test_proxy_config_registers_responses_bridge_for_openai_upstream():\n    \"\"\"A responses-only CLI (codex) hits /v1/responses; a chat-only OpenAI-\n    compatible backend has no /responses endpoint, so the proxy must expose a\n    bridge deployment named ``<model>-responses-bridge`` whose upstream carries\n    the ``openai/chat_completions/`` prefix (LiteLLM strips it and bridges\n    responses→chat). The bridge name is non-slashed (codex mis-parses a slashed\n    model id and then sends no request).\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    by_name = {e[\"model_name\"]: e for e in config[\"model_list\"]}\n    bridge_name = \"gpt-5.4-mini-responses-bridge\"\n    assert bridge_name in by_name\n    assert f\"{route.model_alias}-responses-bridge\" in by_name\n    # the bridge entry's UPSTREAM carries the chat-completions prefix\n    assert (\n        by_name[bridge_name][\"litellm_params\"][\"model\"]\n        == \"openai/chat_completions/gpt-5.4-mini\"\n    )\n    # never slashed (would break codex's model parsing)\n    assert \"/\" not in bridge_name\n\n\ndef test_proxy_config_no_responses_bridge_for_non_openai_upstream():\n    \"\"\"The bridge is openai/-upstream only; native-responses/anthropic/bedrock\n    providers are untouched.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert not any(n.endswith(\"-responses-bridge\") for n in names)\n"}, "86534d47e939a9464112e55b8400964e29b73670": {"t": 1785749195.1725335, "n": 10822, "h": "feac7a868a0591b7", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_openai_route_honors_openhands_reasoning_effort_env():\n    \"\"\"Guards PR #911 against proxy-alias capability guessing.\"\"\"\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.6-sol\",\n        {\n            \"AZURE_API_KEY\": \"key\",\n            \"AZURE_RESOURCE\": \"benchflow\",\n            \"LLM_REASONING_EFFORT\": \"xhigh\",\n        },\n    )\n\n    assert route.litellm_params[\"reasoning_effort\"] == \"xhigh\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\n@pytest.mark.parametrize(\"model\", [\"gemini/gemini-2.5-flash\", \"gemini-2.5-flash\"])\ndef test_gemini_native_route_honors_explicit_base_url(model):\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        model,\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert route.provider_name == \"native\"\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_gemini_native_route_honors_generic_proxy_key():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/BENCHFLOW_PROVIDER_API_KEY\"\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_gemini_native_route_without_explicit_base_url_is_unchanged():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\"GEMINI_API_KEY\": \"sk-gemini\"},\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert \"api_base\" not in route.litellm_params\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_openrouter_route_uses_openai_compatible_endpoint():\n    route = resolve_litellm_route(\n        \"openrouter/qwen/qwen3.5-397b-a17b\",\n        {\"OPENROUTER_API_KEY\": \"sk-openrouter\"},\n    )\n\n    assert route.upstream_model == \"openai/qwen/qwen3.5-397b-a17b\"\n    assert route.litellm_params[\"api_base\"] == \"https://openrouter.ai/api/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/OPENROUTER_API_KEY\"\n    assert route.required_env == (\"OPENROUTER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_forces_chat_completions_for_anthropic_messages():\n    \"\"\"Streaming /v1/messages must bridge via /chat/completions so the\n    LiteLLM success callback fires and llm_trajectory.jsonl is written for\n    claude-agent-acp. LiteLLM's Responses-API streaming adapter (used for\n    openai/-prefixed upstreams such as the vllm provider) skips the success\n    callback; this flag opts out of it (#833).\n    \"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert (\n        config[\"litellm_settings\"][\"use_chat_completions_url_for_anthropic_messages\"]\n        is True\n    )\n\n\ndef test_litellm_exposes_anthropic_messages_chat_completions_flag():\n    \"\"\"Guard against a LiteLLM upgrade silently dropping the flag.\n\n    litellm_proxy_config sets use_chat_completions_url_for_anthropic_messages\n    via LiteLLM's generic litellm_settings -> setattr(litellm, ...) path, which\n    does NOT raise on an unknown key. If a future LiteLLM renames or removes\n    this attribute the fix would become a silent no-op and regress #833, so\n    assert the attribute still exists.\n    \"\"\"\n    import litellm\n\n    assert hasattr(litellm, \"use_chat_completions_url_for_anthropic_messages\")\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n\n\ndef test_proxy_config_registers_responses_bridge_for_openai_upstream():\n    \"\"\"A responses-only CLI (codex) hits /v1/responses; a chat-only OpenAI-\n    compatible backend has no /responses endpoint, so the proxy must expose a\n    bridge deployment named ``<model>-responses-bridge`` whose upstream carries\n    the ``openai/chat_completions/`` prefix (LiteLLM strips it and bridges\n    responses→chat). The bridge name is non-slashed (codex mis-parses a slashed\n    model id and then sends no request).\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    by_name = {e[\"model_name\"]: e for e in config[\"model_list\"]}\n    bridge_name = \"gpt-5.4-mini-responses-bridge\"\n    assert bridge_name in by_name\n    assert f\"{route.model_alias}-responses-bridge\" in by_name\n    # the bridge entry's UPSTREAM carries the chat-completions prefix\n    assert (\n        by_name[bridge_name][\"litellm_params\"][\"model\"]\n        == \"openai/chat_completions/gpt-5.4-mini\"\n    )\n    # never slashed (would break codex's model parsing)\n    assert \"/\" not in bridge_name\n\n\ndef test_proxy_config_no_responses_bridge_for_non_openai_upstream():\n    \"\"\"The bridge is openai/-upstream only; native-responses/anthropic/bedrock\n    providers are untouched.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert not any(n.endswith(\"-responses-bridge\") for n in names)\n"}, "417e3583ecaa5b0f9cce79ad03f72b8a1b9265dc": {"t": 1785749197.293055, "n": 10391, "h": "dfd576734185741a", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\n@pytest.mark.parametrize(\"model\", [\"gemini/gemini-2.5-flash\", \"gemini-2.5-flash\"])\ndef test_gemini_native_route_honors_explicit_base_url(model):\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        model,\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert route.provider_name == \"native\"\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_gemini_native_route_honors_generic_proxy_key():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/BENCHFLOW_PROVIDER_API_KEY\"\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_gemini_native_route_without_explicit_base_url_is_unchanged():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\"GEMINI_API_KEY\": \"sk-gemini\"},\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert \"api_base\" not in route.litellm_params\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_openrouter_route_uses_openai_compatible_endpoint():\n    route = resolve_litellm_route(\n        \"openrouter/qwen/qwen3.5-397b-a17b\",\n        {\"OPENROUTER_API_KEY\": \"sk-openrouter\"},\n    )\n\n    assert route.upstream_model == \"openai/qwen/qwen3.5-397b-a17b\"\n    assert route.litellm_params[\"api_base\"] == \"https://openrouter.ai/api/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/OPENROUTER_API_KEY\"\n    assert route.required_env == (\"OPENROUTER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_forces_chat_completions_for_anthropic_messages():\n    \"\"\"Streaming /v1/messages must bridge via /chat/completions so the\n    LiteLLM success callback fires and llm_trajectory.jsonl is written for\n    claude-agent-acp. LiteLLM's Responses-API streaming adapter (used for\n    openai/-prefixed upstreams such as the vllm provider) skips the success\n    callback; this flag opts out of it (#833).\n    \"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert (\n        config[\"litellm_settings\"][\"use_chat_completions_url_for_anthropic_messages\"]\n        is True\n    )\n\n\ndef test_litellm_exposes_anthropic_messages_chat_completions_flag():\n    \"\"\"Guard against a LiteLLM upgrade silently dropping the flag.\n\n    litellm_proxy_config sets use_chat_completions_url_for_anthropic_messages\n    via LiteLLM's generic litellm_settings -> setattr(litellm, ...) path, which\n    does NOT raise on an unknown key. If a future LiteLLM renames or removes\n    this attribute the fix would become a silent no-op and regress #833, so\n    assert the attribute still exists.\n    \"\"\"\n    import litellm\n\n    assert hasattr(litellm, \"use_chat_completions_url_for_anthropic_messages\")\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n\n\ndef test_proxy_config_registers_responses_bridge_for_openai_upstream():\n    \"\"\"A responses-only CLI (codex) hits /v1/responses; a chat-only OpenAI-\n    compatible backend has no /responses endpoint, so the proxy must expose a\n    bridge deployment named ``<model>-responses-bridge`` whose upstream carries\n    the ``openai/chat_completions/`` prefix (LiteLLM strips it and bridges\n    responses→chat). The bridge name is non-slashed (codex mis-parses a slashed\n    model id and then sends no request).\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    by_name = {e[\"model_name\"]: e for e in config[\"model_list\"]}\n    bridge_name = \"gpt-5.4-mini-responses-bridge\"\n    assert bridge_name in by_name\n    assert f\"{route.model_alias}-responses-bridge\" in by_name\n    # the bridge entry's UPSTREAM carries the chat-completions prefix\n    assert (\n        by_name[bridge_name][\"litellm_params\"][\"model\"]\n        == \"openai/chat_completions/gpt-5.4-mini\"\n    )\n    # never slashed (would break codex's model parsing)\n    assert \"/\" not in bridge_name\n\n\ndef test_proxy_config_no_responses_bridge_for_non_openai_upstream():\n    \"\"\"The bridge is openai/-upstream only; native-responses/anthropic/bedrock\n    providers are untouched.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert not any(n.endswith(\"-responses-bridge\") for n in names)\n"}, "7f590b547ce7be747d62c5b6741594d8d0a9e07f": {"t": 1785749198.4038208, "n": 8558, "h": "680d2331d6acc3cd", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_openrouter_route_uses_openai_compatible_endpoint():\n    route = resolve_litellm_route(\n        \"openrouter/qwen/qwen3.5-397b-a17b\",\n        {\"OPENROUTER_API_KEY\": \"sk-openrouter\"},\n    )\n\n    assert route.upstream_model == \"openai/qwen/qwen3.5-397b-a17b\"\n    assert route.litellm_params[\"api_base\"] == \"https://openrouter.ai/api/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/OPENROUTER_API_KEY\"\n    assert route.required_env == (\"OPENROUTER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_forces_chat_completions_for_anthropic_messages():\n    \"\"\"Streaming /v1/messages must bridge via /chat/completions so the\n    LiteLLM success callback fires and llm_trajectory.jsonl is written for\n    claude-agent-acp. LiteLLM's Responses-API streaming adapter (used for\n    openai/-prefixed upstreams such as the vllm provider) skips the success\n    callback; this flag opts out of it (#833).\n    \"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert (\n        config[\"litellm_settings\"][\"use_chat_completions_url_for_anthropic_messages\"]\n        is True\n    )\n\n\ndef test_litellm_exposes_anthropic_messages_chat_completions_flag():\n    \"\"\"Guard against a LiteLLM upgrade silently dropping the flag.\n\n    litellm_proxy_config sets use_chat_completions_url_for_anthropic_messages\n    via LiteLLM's generic litellm_settings -> setattr(litellm, ...) path, which\n    does NOT raise on an unknown key. If a future LiteLLM renames or removes\n    this attribute the fix would become a silent no-op and regress #833, so\n    assert the attribute still exists.\n    \"\"\"\n    import litellm\n\n    assert hasattr(litellm, \"use_chat_completions_url_for_anthropic_messages\")\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n\n\ndef test_proxy_config_registers_responses_bridge_for_openai_upstream():\n    \"\"\"A responses-only CLI (codex) hits /v1/responses; a chat-only OpenAI-\n    compatible backend has no /responses endpoint, so the proxy must expose a\n    bridge deployment named ``<model>-responses-bridge`` whose upstream carries\n    the ``openai/chat_completions/`` prefix (LiteLLM strips it and bridges\n    responses→chat). The bridge name is non-slashed (codex mis-parses a slashed\n    model id and then sends no request).\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    by_name = {e[\"model_name\"]: e for e in config[\"model_list\"]}\n    bridge_name = \"gpt-5.4-mini-responses-bridge\"\n    assert bridge_name in by_name\n    assert f\"{route.model_alias}-responses-bridge\" in by_name\n    # the bridge entry's UPSTREAM carries the chat-completions prefix\n    assert (\n        by_name[bridge_name][\"litellm_params\"][\"model\"]\n        == \"openai/chat_completions/gpt-5.4-mini\"\n    )\n    # never slashed (would break codex's model parsing)\n    assert \"/\" not in bridge_name\n\n\ndef test_proxy_config_no_responses_bridge_for_non_openai_upstream():\n    \"\"\"The bridge is openai/-upstream only; native-responses/anthropic/bedrock\n    providers are untouched.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert not any(n.endswith(\"-responses-bridge\") for n in names)\n"}, "cc1bf369cac37b058330eaf22dde0d5f350f32cb": {"t": 1785749200.3324397, "n": 6858, "h": "d9bb2d251c45c091", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_openrouter_route_uses_openai_compatible_endpoint():\n    route = resolve_litellm_route(\n        \"openrouter/qwen/qwen3.5-397b-a17b\",\n        {\"OPENROUTER_API_KEY\": \"sk-openrouter\"},\n    )\n\n    assert route.upstream_model == \"openai/qwen/qwen3.5-397b-a17b\"\n    assert route.litellm_params[\"api_base\"] == \"https://openrouter.ai/api/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/OPENROUTER_API_KEY\"\n    assert route.required_env == (\"OPENROUTER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_forces_chat_completions_for_anthropic_messages():\n    \"\"\"Streaming /v1/messages must bridge via /chat/completions so the\n    LiteLLM success callback fires and llm_trajectory.jsonl is written for\n    claude-agent-acp. LiteLLM's Responses-API streaming adapter (used for\n    openai/-prefixed upstreams such as the vllm provider) skips the success\n    callback; this flag opts out of it (#833).\n    \"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert (\n        config[\"litellm_settings\"][\"use_chat_completions_url_for_anthropic_messages\"]\n        is True\n    )\n\n\ndef test_litellm_exposes_anthropic_messages_chat_completions_flag():\n    \"\"\"Guard against a LiteLLM upgrade silently dropping the flag.\n\n    litellm_proxy_config sets use_chat_completions_url_for_anthropic_messages\n    via LiteLLM's generic litellm_settings -> setattr(litellm, ...) path, which\n    does NOT raise on an unknown key. If a future LiteLLM renames or removes\n    this attribute the fix would become a silent no-op and regress #833, so\n    assert the attribute still exists.\n    \"\"\"\n    import litellm\n\n    assert hasattr(litellm, \"use_chat_completions_url_for_anthropic_messages\")\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n"}, "bc429dfbe355d037aee6b8a43ccecd184d25d864": {"t": 1785749201.5757484, "n": 6379, "h": "655a663fa20e1842", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_forces_chat_completions_for_anthropic_messages():\n    \"\"\"Streaming /v1/messages must bridge via /chat/completions so the\n    LiteLLM success callback fires and llm_trajectory.jsonl is written for\n    claude-agent-acp. LiteLLM's Responses-API streaming adapter (used for\n    openai/-prefixed upstreams such as the vllm provider) skips the success\n    callback; this flag opts out of it (#833).\n    \"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert (\n        config[\"litellm_settings\"][\"use_chat_completions_url_for_anthropic_messages\"]\n        is True\n    )\n\n\ndef test_litellm_exposes_anthropic_messages_chat_completions_flag():\n    \"\"\"Guard against a LiteLLM upgrade silently dropping the flag.\n\n    litellm_proxy_config sets use_chat_completions_url_for_anthropic_messages\n    via LiteLLM's generic litellm_settings -> setattr(litellm, ...) path, which\n    does NOT raise on an unknown key. If a future LiteLLM renames or removes\n    this attribute the fix would become a silent no-op and regress #833, so\n    assert the attribute still exists.\n    \"\"\"\n    import litellm\n\n    assert hasattr(litellm, \"use_chat_completions_url_for_anthropic_messages\")\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n"}, "d82754039c4da82024ec6f166154395c1c89848f": {"t": 1785749202.7139218, "n": 5088, "h": "3916a319f88362d3", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n"}, "87b582c6b194794793483bf015afa22bdbf4844d": {"t": 1785749203.9575036, "n": 4979, "h": "a40d4621abb4959f", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n"}, "dd78f835bf8c4db3adb478a14478a99c1f5f4ac0": {"t": 1785749205.1867669, "n": 4678, "h": "775b61e67027c37f", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    assert route.litellm_params[\"reasoning_effort\"] == \"max\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n"}, "66448e07a2950fa92cba19919cb37b367e5a9abf": {"t": 1785749206.7406433, "n": 3989, "h": "890b2f83820d3081", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    assert route.litellm_params[\"reasoning_effort\"] == \"max\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n"}, "415e8ae093e9e0bf7ba29d43ac71abe34a39e5ee": {"t": 1785749207.8075092, "n": 3454, "h": "f84d7bf73843b137", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    assert route.litellm_params[\"reasoning_effort\"] == \"max\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n"}, "2631b1d47436fbaa6017956c2957e7d4147b3cae": {"t": 1786068514.0561101, "n": 10822, "h": "feac7a868a0591b7", "c": "from __future__ import annotations\n\nimport pytest\n\nfrom benchflow.providers.litellm_config import (\n    litellm_proxy_config,\n    resolve_litellm_route,\n)\n\n\ndef test_bedrock_model_maps_to_litellm_bedrock_route():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n\n    assert route.model_alias == \"benchflow-aws-bedrock-us.anthropic.claude-opus-4-8\"\n    assert route.upstream_model == \"bedrock/us.anthropic.claude-opus-4-8\"\n    assert route.required_env == (\"AWS_BEARER_TOKEN_BEDROCK\", \"AWS_REGION\")\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_bedrock_model_honors_max_thinking_effort_env():\n    \"\"\"Guards PR #739 against #737's route-config effort ceiling regression.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\n            \"AWS_BEARER_TOKEN_BEDROCK\": \"token\",\n            \"AWS_REGION\": \"us-west-2\",\n            \"BENCHFLOW_BEDROCK_THINKING_EFFORT\": \"max\",\n        },\n    )\n\n    # `max` is honored as \"the highest supported effort\": LiteLLM 1.88.0rc1\n    # rejects `max`/`xhigh` for opus-4-8, so BenchFlow clamps to the accepted\n    # ceiling `high` rather than erroring at request time (#737).\n    assert route.litellm_params[\"reasoning_effort\"] == \"high\"\n\n\ndef test_azure_openai_route_uses_resource_and_preview_version():\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure/gpt-5.5\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/AZURE_API_KEY\"\n    assert route.litellm_params[\"api_base\"] == \"https://benchflow.openai.azure.com/\"\n    assert route.litellm_params[\"api_version\"] == \"preview\"\n\n\ndef test_azure_openai_route_honors_openhands_reasoning_effort_env():\n    \"\"\"Guards PR #911 against proxy-alias capability guessing.\"\"\"\n    route = resolve_litellm_route(\n        \"azure-foundry-openai/gpt-5.6-sol\",\n        {\n            \"AZURE_API_KEY\": \"key\",\n            \"AZURE_RESOURCE\": \"benchflow\",\n            \"LLM_REASONING_EFFORT\": \"xhigh\",\n        },\n    )\n\n    assert route.litellm_params[\"reasoning_effort\"] == \"xhigh\"\n\n\ndef test_azure_anthropic_route_uses_azure_ai_anthropic_surface():\n    route = resolve_litellm_route(\n        \"azure-foundry-anthropic/claude-opus-4-5\",\n        {\"AZURE_API_KEY\": \"key\", \"AZURE_RESOURCE\": \"benchflow\"},\n    )\n\n    assert route.upstream_model == \"azure_ai/claude-opus-4-5\"\n    assert (\n        route.litellm_params[\"api_base\"]\n        == \"https://benchflow.services.ai.azure.com/anthropic\"\n    )\n\n\n@pytest.mark.parametrize(\n    (\"model\", \"upstream\", \"required_env\"),\n    [\n        (\"openai/gpt-4.1-mini\", \"openai/gpt-4.1-mini\", (\"OPENAI_API_KEY\",)),\n        (\"claude-sonnet-4-6\", \"anthropic/claude-sonnet-4-6\", (\"ANTHROPIC_API_KEY\",)),\n        (\"gemini-3.5-flash\", \"gemini/gemini-3.5-flash\", (\"GEMINI_API_KEY\",)),\n        (\"minimax/MiniMax-M3\", \"openai/MiniMax-M3\", (\"MINIMAX_API_KEY\",)),\n    ],\n)\ndef test_common_provider_routes(model, upstream, required_env):\n    route = resolve_litellm_route(\n        model,\n        {\n            \"OPENAI_API_KEY\": \"openai\",\n            \"ANTHROPIC_API_KEY\": \"anthropic\",\n            \"GEMINI_API_KEY\": \"gemini\",\n            \"MINIMAX_API_KEY\": \"minimax\",\n            \"MINIMAX_BASE_URL\": \"https://api.minimax.io/v1\",\n        },\n    )\n\n    assert route.upstream_model == upstream\n    assert route.required_env == required_env\n\n\ndef test_registered_provider_route_honors_explicit_generic_proxy_env():\n    \"\"\"Guards PR #780: external LiteLLM proxies can back registered providers.\"\"\"\n    route = resolve_litellm_route(\n        \"deepseek/deepseek-v4-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://llm-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n        },\n    )\n\n    assert route.upstream_model == \"openai/deepseek-v4-flash\"\n    assert route.litellm_params[\"api_base\"] == \"https://llm-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == (\"os.environ/BENCHFLOW_PROVIDER_API_KEY\")\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\n@pytest.mark.parametrize(\"model\", [\"gemini/gemini-2.5-flash\", \"gemini-2.5-flash\"])\ndef test_gemini_native_route_honors_explicit_base_url(model):\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        model,\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert route.provider_name == \"native\"\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_gemini_native_route_honors_generic_proxy_key():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\n            \"BENCHFLOW_PROVIDER_BASE_URL\": \"https://gemini-proxy.example.test/v1\",\n            \"BENCHFLOW_PROVIDER_API_KEY\": \"sk-proxy\",\n            \"GEMINI_API_KEY\": \"sk-gemini\",\n        },\n    )\n\n    assert route.litellm_params[\"api_base\"] == \"https://gemini-proxy.example.test/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/BENCHFLOW_PROVIDER_API_KEY\"\n    assert route.required_env == (\"BENCHFLOW_PROVIDER_API_KEY\",)\n\n\ndef test_gemini_native_route_without_explicit_base_url_is_unchanged():\n    \"\"\"Guards the fix from PR #881 for issue #672.\"\"\"\n    route = resolve_litellm_route(\n        \"gemini/gemini-2.5-flash\",\n        {\"GEMINI_API_KEY\": \"sk-gemini\"},\n    )\n\n    assert route.upstream_model == \"gemini/gemini-2.5-flash\"\n    assert \"api_base\" not in route.litellm_params\n    assert route.litellm_params[\"api_key\"] == \"os.environ/GEMINI_API_KEY\"\n    assert route.required_env == (\"GEMINI_API_KEY\",)\n\n\ndef test_openrouter_route_uses_openai_compatible_endpoint():\n    route = resolve_litellm_route(\n        \"openrouter/qwen/qwen3.5-397b-a17b\",\n        {\"OPENROUTER_API_KEY\": \"sk-openrouter\"},\n    )\n\n    assert route.upstream_model == \"openai/qwen/qwen3.5-397b-a17b\"\n    assert route.litellm_params[\"api_base\"] == \"https://openrouter.ai/api/v1\"\n    assert route.litellm_params[\"api_key\"] == \"os.environ/OPENROUTER_API_KEY\"\n    assert route.required_env == (\"OPENROUTER_API_KEY\",)\n\n\ndef test_proxy_config_registers_plain_and_openai_aliases():\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert config[\"general_settings\"] == {\"master_key\": \"sk-local\"}\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert route.model_alias in names\n    assert f\"openai/{route.model_alias}\" in names\n    assert \"us.anthropic.claude-opus-4-8\" in names\n    assert \"openai/us.anthropic.claude-opus-4-8\" in names\n    assert config[\"litellm_settings\"][\"callbacks\"] == [\n        \"benchflow_litellm_callback.proxy_handler_instance\"\n    ]\n    assert config[\"router_settings\"] == {\n        \"num_retries\": 0,\n        \"disable_cooldowns\": True,\n    }\n\n\ndef test_proxy_config_forces_chat_completions_for_anthropic_messages():\n    \"\"\"Streaming /v1/messages must bridge via /chat/completions so the\n    LiteLLM success callback fires and llm_trajectory.jsonl is written for\n    claude-agent-acp. LiteLLM's Responses-API streaming adapter (used for\n    openai/-prefixed upstreams such as the vllm provider) skips the success\n    callback; this flag opts out of it (#833).\n    \"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    assert (\n        config[\"litellm_settings\"][\"use_chat_completions_url_for_anthropic_messages\"]\n        is True\n    )\n\n\ndef test_litellm_exposes_anthropic_messages_chat_completions_flag():\n    \"\"\"Guard against a LiteLLM upgrade silently dropping the flag.\n\n    litellm_proxy_config sets use_chat_completions_url_for_anthropic_messages\n    via LiteLLM's generic litellm_settings -> setattr(litellm, ...) path, which\n    does NOT raise on an unknown key. If a future LiteLLM renames or removes\n    this attribute the fix would become a silent no-op and regress #833, so\n    assert the attribute still exists.\n    \"\"\"\n    import litellm\n\n    assert hasattr(litellm, \"use_chat_completions_url_for_anthropic_messages\")\n\n\ndef test_proxy_config_registers_requested_bare_model_name():\n    \"\"\"Codex ACP sends the bare selected model name to the proxy.\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert \"gpt-5.4-mini\" in names\n    assert \"openai/gpt-5.4-mini\" in names\n\n\ndef test_proxy_config_registers_responses_bridge_for_openai_upstream():\n    \"\"\"A responses-only CLI (codex) hits /v1/responses; a chat-only OpenAI-\n    compatible backend has no /responses endpoint, so the proxy must expose a\n    bridge deployment named ``<model>-responses-bridge`` whose upstream carries\n    the ``openai/chat_completions/`` prefix (LiteLLM strips it and bridges\n    responses→chat). The bridge name is non-slashed (codex mis-parses a slashed\n    model id and then sends no request).\"\"\"\n    route = resolve_litellm_route(\"openai/gpt-5.4-mini\", {\"OPENAI_API_KEY\": \"key\"})\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n\n    by_name = {e[\"model_name\"]: e for e in config[\"model_list\"]}\n    bridge_name = \"gpt-5.4-mini-responses-bridge\"\n    assert bridge_name in by_name\n    assert f\"{route.model_alias}-responses-bridge\" in by_name\n    # the bridge entry's UPSTREAM carries the chat-completions prefix\n    assert (\n        by_name[bridge_name][\"litellm_params\"][\"model\"]\n        == \"openai/chat_completions/gpt-5.4-mini\"\n    )\n    # never slashed (would break codex's model parsing)\n    assert \"/\" not in bridge_name\n\n\ndef test_proxy_config_no_responses_bridge_for_non_openai_upstream():\n    \"\"\"The bridge is openai/-upstream only; native-responses/anthropic/bedrock\n    providers are untouched.\"\"\"\n    route = resolve_litellm_route(\n        \"aws-bedrock/us.anthropic.claude-opus-4-8\",\n        {\"AWS_BEARER_TOKEN_BEDROCK\": \"token\", \"AWS_REGION\": \"us-west-2\"},\n    )\n    config = litellm_proxy_config(route, master_key=\"sk-local\")\n    names = [entry[\"model_name\"] for entry in config[\"model_list\"]]\n    assert not any(n.endswith(\"-responses-bridge\") for n in names)\n"}}, ".github/scripts/setup_codex_deepseek_bridge.sh": {"cf656179f3bc30b2887b744cf71b69b5e5ffa5ff": {"t": 1785747012.5737393, "n": 4752, "h": "c94518e67ad55232", "c": "#!/usr/bin/env bash\n# Set up Moon Bridge so the L3/L2 codex equivalence reviewer can run on\n# DeepSeek-v4-pro. codex speaks ONLY the OpenAI Responses API; Moon Bridge is a\n# local proxy that exposes a Responses-compatible endpoint and forwards to\n# DeepSeek's /anthropic surface (validated end-to-end). Method:\n# https://github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md\n#\n# Starts the bridge in the BACKGROUND (it must stay alive for the later codex\n# step) and writes codex's config.toml into $CODEX_HOME so `codex exec` routes to\n# DeepSeek. Requires $DEEPSEEK_API_KEY in the env. Idempotent-ish; fail-closed.\nset -euo pipefail\n\n: \"${DEEPSEEK_API_KEY:?DEEPSEEK_API_KEY required for the codex DeepSeek bridge}\"\nGO_VERSION=\"${GO_VERSION:-go1.25.0}\"\nBRIDGE_ADDR=\"${BRIDGE_ADDR:-127.0.0.1:38440}\"\nHERE=\"$(cd \"$(dirname \"$0\")\" && pwd)\"\nCONFIG_TEMPLATE=\"${HERE}/../integration/moonbridge-config.yml\"\n\n# 1. Go toolchain (Moon Bridge needs >= 1.25).\nif ! command -v go >/dev/null 2>&1 || ! go version | grep -qE 'go1\\.(2[5-9]|[3-9][0-9])'; then\n  echo \"installing ${GO_VERSION}...\"\n  curl -fsSL \"https://go.dev/dl/${GO_VERSION}.linux-amd64.tar.gz\" | tar -C /tmp -xz\n  export PATH=\"/tmp/go/bin:${PATH}\"\nfi\nexport GOPATH=\"${GOPATH:-/tmp/gopath}\" GOCACHE=\"${GOCACHE:-/tmp/gocache}\"\n\n# 2. Build Moon Bridge at a PINNED, locally-reviewed commit. A git SHA is\n#    content-addressed and immutable, so a later malicious/force-pushed commit to\n#    the upstream repo can NEVER change what this SHA builds — closes the\n#    supply-chain gap of cloning an unpinned HEAD into the trusted review-pack job.\n#    Fetch only that one commit (shallow). Bump this SHA only after re-reviewing.\nMOONBRIDGE_REF=\"${MOONBRIDGE_REF:-8254b4148cefd54828b2aec37bb8b0c7ad9e5cb6}\"\nrm -rf /tmp/moon-bridge\ngit -c advice.detachedHead=false init -q /tmp/moon-bridge\ngit -C /tmp/moon-bridge remote add origin https://github.com/ZhiYi-R/moon-bridge.git\ngit -C /tmp/moon-bridge fetch -q --depth 1 origin \"${MOONBRIDGE_REF}\"\ngit -C /tmp/moon-bridge -c advice.detachedHead=false checkout -q \"${MOONBRIDGE_REF}\"\ngot=\"$(git -C /tmp/moon-bridge rev-parse HEAD)\"\nif [ \"${got}\" != \"${MOONBRIDGE_REF}\" ]; then\n  echo \"::error::moon-bridge checkout ${got} != pinned ${MOONBRIDGE_REF}\"; exit 1\nfi\n( cd /tmp/moon-bridge && go build -o /tmp/moonbridge ./cmd/moonbridge )\n\n# 3. config.yml with the live DeepSeek key. Substitute injection-SAFELY: the key\n#    is read from the env (never a shell/sed arg) and JSON-encoded (a valid,\n#    properly-escaped YAML string), so a key containing the sed delimiter, quotes,\n#    backslashes, or YAML-active chars cannot corrupt the config or inject.\nDS_TEMPLATE=\"${CONFIG_TEMPLATE}\" DS_OUT=/tmp/mb_config.yml python3 - <<'PY'\nimport json, os, pathlib\ntpl = pathlib.Path(os.environ[\"DS_TEMPLATE\"]).read_text()\nkey = os.environ[\"DEEPSEEK_API_KEY\"]\n# Match the QUOTED placeholder (the api_key value slot) and replace it with the\n# JSON-encoded (YAML-safe) key. We match the quoted form, not the bare word,\n# because the word also appears in a YAML comment in the template — matching the\n# quoted form leaves that comment untouched.\ntoken = '\"DEEPSEEK_API_KEY_PLACEHOLDER\"'\nif token not in tpl:\n    raise SystemExit(\"api_key placeholder token not found in template\")\nout = tpl.replace(token, json.dumps(key))\nif token in out:  # the quoted secret slot must be gone after substitution\n    raise SystemExit(\"placeholder substitution failed\")\npathlib.Path(os.environ[\"DS_OUT\"]).write_text(out)\nPY\n\n# 4. Start the bridge in the background; wait until it accepts connections.\nnohup /tmp/moonbridge -config /tmp/mb_config.yml > /tmp/moonbridge.log 2>&1 < /dev/null &\ndisown || true\nfor _ in $(seq 1 30); do\n  curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1 && break\n  sleep 1\ndone\nif ! curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1; then\n  echo \"::error::Moon Bridge did not start\"\n  tail -30 /tmp/moonbridge.log || true\n  exit 1\nfi\n\n# 5. Generate codex config.toml (provider=moonbridge, wire_api=responses) into\n#    $CODEX_HOME so `codex exec --model moonbridge` routes through the bridge.\nCODEX_HOME=\"${CODEX_HOME:-${HOME}/.codex}\"\nmkdir -p \"${CODEX_HOME}\"\nMODEL=\"$(/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-model)\"\n/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-config \"${MODEL}\" \\\n  -codex-base-url \"http://${BRIDGE_ADDR}/v1\" -codex-home \"${CODEX_HOME}\" \\\n  > \"${CODEX_HOME}/config.toml\"\n# The deepwiki MCP server the generator adds is unused by the read-only reviewer\n# and only adds startup latency — drop it.\nif grep -q '\\[mcp_servers' \"${CODEX_HOME}/config.toml\"; then\n  sed -i '/^\\[mcp_servers/,/^$/d' \"${CODEX_HOME}/config.toml\"\nfi\necho \"codex-deepseek bridge ready: codex model=${MODEL} -> deepseek-v4-pro\"\n"}, "6f023e05819aa391092d548bbf9f5925651d6365": {"t": 1785749195.8334055, "n": 4752, "h": "c94518e67ad55232", "c": "#!/usr/bin/env bash\n# Set up Moon Bridge so the L3/L2 codex equivalence reviewer can run on\n# DeepSeek-v4-pro. codex speaks ONLY the OpenAI Responses API; Moon Bridge is a\n# local proxy that exposes a Responses-compatible endpoint and forwards to\n# DeepSeek's /anthropic surface (validated end-to-end). Method:\n# https://github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md\n#\n# Starts the bridge in the BACKGROUND (it must stay alive for the later codex\n# step) and writes codex's config.toml into $CODEX_HOME so `codex exec` routes to\n# DeepSeek. Requires $DEEPSEEK_API_KEY in the env. Idempotent-ish; fail-closed.\nset -euo pipefail\n\n: \"${DEEPSEEK_API_KEY:?DEEPSEEK_API_KEY required for the codex DeepSeek bridge}\"\nGO_VERSION=\"${GO_VERSION:-go1.25.0}\"\nBRIDGE_ADDR=\"${BRIDGE_ADDR:-127.0.0.1:38440}\"\nHERE=\"$(cd \"$(dirname \"$0\")\" && pwd)\"\nCONFIG_TEMPLATE=\"${HERE}/../integration/moonbridge-config.yml\"\n\n# 1. Go toolchain (Moon Bridge needs >= 1.25).\nif ! command -v go >/dev/null 2>&1 || ! go version | grep -qE 'go1\\.(2[5-9]|[3-9][0-9])'; then\n  echo \"installing ${GO_VERSION}...\"\n  curl -fsSL \"https://go.dev/dl/${GO_VERSION}.linux-amd64.tar.gz\" | tar -C /tmp -xz\n  export PATH=\"/tmp/go/bin:${PATH}\"\nfi\nexport GOPATH=\"${GOPATH:-/tmp/gopath}\" GOCACHE=\"${GOCACHE:-/tmp/gocache}\"\n\n# 2. Build Moon Bridge at a PINNED, locally-reviewed commit. A git SHA is\n#    content-addressed and immutable, so a later malicious/force-pushed commit to\n#    the upstream repo can NEVER change what this SHA builds — closes the\n#    supply-chain gap of cloning an unpinned HEAD into the trusted review-pack job.\n#    Fetch only that one commit (shallow). Bump this SHA only after re-reviewing.\nMOONBRIDGE_REF=\"${MOONBRIDGE_REF:-8254b4148cefd54828b2aec37bb8b0c7ad9e5cb6}\"\nrm -rf /tmp/moon-bridge\ngit -c advice.detachedHead=false init -q /tmp/moon-bridge\ngit -C /tmp/moon-bridge remote add origin https://github.com/ZhiYi-R/moon-bridge.git\ngit -C /tmp/moon-bridge fetch -q --depth 1 origin \"${MOONBRIDGE_REF}\"\ngit -C /tmp/moon-bridge -c advice.detachedHead=false checkout -q \"${MOONBRIDGE_REF}\"\ngot=\"$(git -C /tmp/moon-bridge rev-parse HEAD)\"\nif [ \"${got}\" != \"${MOONBRIDGE_REF}\" ]; then\n  echo \"::error::moon-bridge checkout ${got} != pinned ${MOONBRIDGE_REF}\"; exit 1\nfi\n( cd /tmp/moon-bridge && go build -o /tmp/moonbridge ./cmd/moonbridge )\n\n# 3. config.yml with the live DeepSeek key. Substitute injection-SAFELY: the key\n#    is read from the env (never a shell/sed arg) and JSON-encoded (a valid,\n#    properly-escaped YAML string), so a key containing the sed delimiter, quotes,\n#    backslashes, or YAML-active chars cannot corrupt the config or inject.\nDS_TEMPLATE=\"${CONFIG_TEMPLATE}\" DS_OUT=/tmp/mb_config.yml python3 - <<'PY'\nimport json, os, pathlib\ntpl = pathlib.Path(os.environ[\"DS_TEMPLATE\"]).read_text()\nkey = os.environ[\"DEEPSEEK_API_KEY\"]\n# Match the QUOTED placeholder (the api_key value slot) and replace it with the\n# JSON-encoded (YAML-safe) key. We match the quoted form, not the bare word,\n# because the word also appears in a YAML comment in the template — matching the\n# quoted form leaves that comment untouched.\ntoken = '\"DEEPSEEK_API_KEY_PLACEHOLDER\"'\nif token not in tpl:\n    raise SystemExit(\"api_key placeholder token not found in template\")\nout = tpl.replace(token, json.dumps(key))\nif token in out:  # the quoted secret slot must be gone after substitution\n    raise SystemExit(\"placeholder substitution failed\")\npathlib.Path(os.environ[\"DS_OUT\"]).write_text(out)\nPY\n\n# 4. Start the bridge in the background; wait until it accepts connections.\nnohup /tmp/moonbridge -config /tmp/mb_config.yml > /tmp/moonbridge.log 2>&1 < /dev/null &\ndisown || true\nfor _ in $(seq 1 30); do\n  curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1 && break\n  sleep 1\ndone\nif ! curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1; then\n  echo \"::error::Moon Bridge did not start\"\n  tail -30 /tmp/moonbridge.log || true\n  exit 1\nfi\n\n# 5. Generate codex config.toml (provider=moonbridge, wire_api=responses) into\n#    $CODEX_HOME so `codex exec --model moonbridge` routes through the bridge.\nCODEX_HOME=\"${CODEX_HOME:-${HOME}/.codex}\"\nmkdir -p \"${CODEX_HOME}\"\nMODEL=\"$(/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-model)\"\n/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-config \"${MODEL}\" \\\n  -codex-base-url \"http://${BRIDGE_ADDR}/v1\" -codex-home \"${CODEX_HOME}\" \\\n  > \"${CODEX_HOME}/config.toml\"\n# The deepwiki MCP server the generator adds is unused by the read-only reviewer\n# and only adds startup latency — drop it.\nif grep -q '\\[mcp_servers' \"${CODEX_HOME}/config.toml\"; then\n  sed -i '/^\\[mcp_servers/,/^$/d' \"${CODEX_HOME}/config.toml\"\nfi\necho \"codex-deepseek bridge ready: codex model=${MODEL} -> deepseek-v4-pro\"\n"}, "3daf698643c7db5fecf1432da3fb1bf2c837e46c": {"t": 1785749216.479533, "n": 2906, "h": "b1a7b63a63b561a7", "c": "#!/usr/bin/env bash\n# Set up Moon Bridge so the L3/L2 codex equivalence reviewer can run on\n# DeepSeek-v4-pro. codex speaks ONLY the OpenAI Responses API; Moon Bridge is a\n# local proxy that exposes a Responses-compatible endpoint and forwards to\n# DeepSeek's /anthropic surface (validated end-to-end). Method:\n# https://github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md\n#\n# Starts the bridge in the BACKGROUND (it must stay alive for the later codex\n# step) and writes codex's config.toml into $CODEX_HOME so `codex exec` routes to\n# DeepSeek. Requires $DEEPSEEK_API_KEY in the env. Idempotent-ish; fail-closed.\nset -euo pipefail\n\n: \"${DEEPSEEK_API_KEY:?DEEPSEEK_API_KEY required for the codex DeepSeek bridge}\"\nGO_VERSION=\"${GO_VERSION:-go1.25.0}\"\nBRIDGE_ADDR=\"${BRIDGE_ADDR:-127.0.0.1:38440}\"\nHERE=\"$(cd \"$(dirname \"$0\")\" && pwd)\"\nCONFIG_TEMPLATE=\"${HERE}/../integration/moonbridge-config.yml\"\n\n# 1. Go toolchain (Moon Bridge needs >= 1.25).\nif ! command -v go >/dev/null 2>&1 || ! go version | grep -qE 'go1\\.(2[5-9]|[3-9][0-9])'; then\n  echo \"installing ${GO_VERSION}...\"\n  curl -fsSL \"https://go.dev/dl/${GO_VERSION}.linux-amd64.tar.gz\" | tar -C /tmp -xz\n  export PATH=\"/tmp/go/bin:${PATH}\"\nfi\nexport GOPATH=\"${GOPATH:-/tmp/gopath}\" GOCACHE=\"${GOCACHE:-/tmp/gocache}\"\n\n# 2. Build Moon Bridge.\nrm -rf /tmp/moon-bridge\ngit clone --depth 1 https://github.com/ZhiYi-R/moon-bridge.git /tmp/moon-bridge\n( cd /tmp/moon-bridge && go build -o /tmp/moonbridge ./cmd/moonbridge )\n\n# 3. config.yml with the live DeepSeek key (placeholder -> $DEEPSEEK_API_KEY).\nsed \"s|DEEPSEEK_API_KEY_PLACEHOLDER|${DEEPSEEK_API_KEY}|\" \\\n  \"${CONFIG_TEMPLATE}\" > /tmp/mb_config.yml\n\n# 4. Start the bridge in the background; wait until it accepts connections.\nnohup /tmp/moonbridge -config /tmp/mb_config.yml > /tmp/moonbridge.log 2>&1 &\nfor _ in $(seq 1 30); do\n  curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1 && break\n  sleep 1\ndone\nif ! curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1; then\n  echo \"::error::Moon Bridge did not start\"\n  tail -30 /tmp/moonbridge.log || true\n  exit 1\nfi\n\n# 5. Generate codex config.toml (provider=moonbridge, wire_api=responses) into\n#    $CODEX_HOME so `codex exec --model moonbridge` routes through the bridge.\nCODEX_HOME=\"${CODEX_HOME:-${HOME}/.codex}\"\nmkdir -p \"${CODEX_HOME}\"\nMODEL=\"$(/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-model)\"\n/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-config \"${MODEL}\" \\\n  -codex-base-url \"http://${BRIDGE_ADDR}/v1\" -codex-home \"${CODEX_HOME}\" \\\n  > \"${CODEX_HOME}/config.toml\"\n# The deepwiki MCP server the generator adds is unused by the read-only reviewer\n# and only adds startup latency — drop it.\nif grep -q '\\[mcp_servers' \"${CODEX_HOME}/config.toml\"; then\n  sed -i '/^\\[mcp_servers/,/^$/d' \"${CODEX_HOME}/config.toml\"\nfi\necho \"codex-deepseek bridge ready: codex model=${MODEL} -> deepseek-v4-pro\"\n"}, "2631b1d47436fbaa6017956c2957e7d4147b3cae": {"t": 1786068535.1738527, "n": 4752, "h": "c94518e67ad55232", "c": "#!/usr/bin/env bash\n# Set up Moon Bridge so the L3/L2 codex equivalence reviewer can run on\n# DeepSeek-v4-pro. codex speaks ONLY the OpenAI Responses API; Moon Bridge is a\n# local proxy that exposes a Responses-compatible endpoint and forwards to\n# DeepSeek's /anthropic surface (validated end-to-end). Method:\n# https://github.com/deepseek-ai/awesome-deepseek-agent docs/codex.md\n#\n# Starts the bridge in the BACKGROUND (it must stay alive for the later codex\n# step) and writes codex's config.toml into $CODEX_HOME so `codex exec` routes to\n# DeepSeek. Requires $DEEPSEEK_API_KEY in the env. Idempotent-ish; fail-closed.\nset -euo pipefail\n\n: \"${DEEPSEEK_API_KEY:?DEEPSEEK_API_KEY required for the codex DeepSeek bridge}\"\nGO_VERSION=\"${GO_VERSION:-go1.25.0}\"\nBRIDGE_ADDR=\"${BRIDGE_ADDR:-127.0.0.1:38440}\"\nHERE=\"$(cd \"$(dirname \"$0\")\" && pwd)\"\nCONFIG_TEMPLATE=\"${HERE}/../integration/moonbridge-config.yml\"\n\n# 1. Go toolchain (Moon Bridge needs >= 1.25).\nif ! command -v go >/dev/null 2>&1 || ! go version | grep -qE 'go1\\.(2[5-9]|[3-9][0-9])'; then\n  echo \"installing ${GO_VERSION}...\"\n  curl -fsSL \"https://go.dev/dl/${GO_VERSION}.linux-amd64.tar.gz\" | tar -C /tmp -xz\n  export PATH=\"/tmp/go/bin:${PATH}\"\nfi\nexport GOPATH=\"${GOPATH:-/tmp/gopath}\" GOCACHE=\"${GOCACHE:-/tmp/gocache}\"\n\n# 2. Build Moon Bridge at a PINNED, locally-reviewed commit. A git SHA is\n#    content-addressed and immutable, so a later malicious/force-pushed commit to\n#    the upstream repo can NEVER change what this SHA builds — closes the\n#    supply-chain gap of cloning an unpinned HEAD into the trusted review-pack job.\n#    Fetch only that one commit (shallow). Bump this SHA only after re-reviewing.\nMOONBRIDGE_REF=\"${MOONBRIDGE_REF:-8254b4148cefd54828b2aec37bb8b0c7ad9e5cb6}\"\nrm -rf /tmp/moon-bridge\ngit -c advice.detachedHead=false init -q /tmp/moon-bridge\ngit -C /tmp/moon-bridge remote add origin https://github.com/ZhiYi-R/moon-bridge.git\ngit -C /tmp/moon-bridge fetch -q --depth 1 origin \"${MOONBRIDGE_REF}\"\ngit -C /tmp/moon-bridge -c advice.detachedHead=false checkout -q \"${MOONBRIDGE_REF}\"\ngot=\"$(git -C /tmp/moon-bridge rev-parse HEAD)\"\nif [ \"${got}\" != \"${MOONBRIDGE_REF}\" ]; then\n  echo \"::error::moon-bridge checkout ${got} != pinned ${MOONBRIDGE_REF}\"; exit 1\nfi\n( cd /tmp/moon-bridge && go build -o /tmp/moonbridge ./cmd/moonbridge )\n\n# 3. config.yml with the live DeepSeek key. Substitute injection-SAFELY: the key\n#    is read from the env (never a shell/sed arg) and JSON-encoded (a valid,\n#    properly-escaped YAML string), so a key containing the sed delimiter, quotes,\n#    backslashes, or YAML-active chars cannot corrupt the config or inject.\nDS_TEMPLATE=\"${CONFIG_TEMPLATE}\" DS_OUT=/tmp/mb_config.yml python3 - <<'PY'\nimport json, os, pathlib\ntpl = pathlib.Path(os.environ[\"DS_TEMPLATE\"]).read_text()\nkey = os.environ[\"DEEPSEEK_API_KEY\"]\n# Match the QUOTED placeholder (the api_key value slot) and replace it with the\n# JSON-encoded (YAML-safe) key. We match the quoted form, not the bare word,\n# because the word also appears in a YAML comment in the template — matching the\n# quoted form leaves that comment untouched.\ntoken = '\"DEEPSEEK_API_KEY_PLACEHOLDER\"'\nif token not in tpl:\n    raise SystemExit(\"api_key placeholder token not found in template\")\nout = tpl.replace(token, json.dumps(key))\nif token in out:  # the quoted secret slot must be gone after substitution\n    raise SystemExit(\"placeholder substitution failed\")\npathlib.Path(os.environ[\"DS_OUT\"]).write_text(out)\nPY\n\n# 4. Start the bridge in the background; wait until it accepts connections.\nnohup /tmp/moonbridge -config /tmp/mb_config.yml > /tmp/moonbridge.log 2>&1 < /dev/null &\ndisown || true\nfor _ in $(seq 1 30); do\n  curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1 && break\n  sleep 1\ndone\nif ! curl -sf \"http://${BRIDGE_ADDR}/console/\" >/dev/null 2>&1; then\n  echo \"::error::Moon Bridge did not start\"\n  tail -30 /tmp/moonbridge.log || true\n  exit 1\nfi\n\n# 5. Generate codex config.toml (provider=moonbridge, wire_api=responses) into\n#    $CODEX_HOME so `codex exec --model moonbridge` routes through the bridge.\nCODEX_HOME=\"${CODEX_HOME:-${HOME}/.codex}\"\nmkdir -p \"${CODEX_HOME}\"\nMODEL=\"$(/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-model)\"\n/tmp/moonbridge -config /tmp/mb_config.yml -print-codex-config \"${MODEL}\" \\\n  -codex-base-url \"http://${BRIDGE_ADDR}/v1\" -codex-home \"${CODEX_HOME}\" \\\n  > \"${CODEX_HOME}/config.toml\"\n# The deepwiki MCP server the generator adds is unused by the read-only reviewer\n# and only adds startup latency — drop it.\nif grep -q '\\[mcp_servers' \"${CODEX_HOME}/config.toml\"; then\n  sed -i '/^\\[mcp_servers/,/^$/d' \"${CODEX_HOME}/config.toml\"\nfi\necho \"codex-deepseek bridge ready: codex model=${MODEL} -> deepseek-v4-pro\"\n"}}, ".github/scripts/select_integration_provider.py": {"cf656179f3bc30b2887b744cf71b69b5e5ffa5ff": {"t": 1785747012.5962758, "n": 8358, "h": "1868a983378a1658", "c": "#!/usr/bin/env python3\n\"\"\"Select a live model provider for the integration-eval workflow.\n\nThe workflow needs a real LLM for both the rollout and the agent-as-judge\ngate. Repo secrets can drift independently from code, so this script probes the\nconfigured candidates with a tiny OpenAI-compatible request and exports the\nfirst usable one to ``GITHUB_ENV``. It prints only candidate names and HTTP\nstatus classes, never secret values or response bodies.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nimport ssl\nimport sys\nimport urllib.error\nimport urllib.request\nfrom collections.abc import Callable, Mapping\nfrom dataclasses import dataclass\nfrom pathlib import Path\n\nProbeFunc = Callable[[\"Candidate\", Mapping[str, str]], tuple[bool, str]]\n\n\n@dataclass(frozen=True)\nclass Candidate:\n    name: str\n    key_env: str\n    base_url: str\n    probe_model: str\n    rollout_agent: str\n    rollout_model: str\n    judge_model: str\n    exports: Mapping[str, str]\n    insecure_tls: bool = False\n\n\ndef _append_github_env(values: Mapping[str, str], env: Mapping[str, str]) -> None:\n    path = env.get(\"GITHUB_ENV\")\n    if not path:\n        return\n    marker = \"__BENCHFLOW_ENV__\"\n    with Path(path).open(\"a\", encoding=\"utf-8\") as fh:\n        for key, value in values.items():\n            fh.write(f\"{key}<<{marker}\\n{value}\\n{marker}\\n\")\n\n\ndef _chat_completions_url(base_url: str) -> str:\n    return f\"{base_url.rstrip('/')}/chat/completions\"\n\n\ndef probe_candidate(candidate: Candidate, env: Mapping[str, str]) -> tuple[bool, str]:\n    key = env.get(candidate.key_env, \"\").strip()\n    if not key:\n        return False, f\"missing {candidate.key_env}\"\n\n    body = json.dumps(\n        {\n            \"model\": candidate.probe_model,\n            \"messages\": [{\"role\": \"user\", \"content\": \"Return OK only.\"}],\n            \"max_tokens\": 8,\n        }\n    ).encode(\"utf-8\")\n    request = urllib.request.Request(\n        _chat_completions_url(candidate.base_url),\n        data=body,\n        headers={\n            \"Authorization\": f\"Bearer {key}\",\n            \"Content-Type\": \"application/json\",\n        },\n        method=\"POST\",\n    )\n    context = None\n    if candidate.insecure_tls:\n        context = ssl._create_unverified_context()\n    try:\n        with urllib.request.urlopen(request, timeout=25, context=context) as response:\n            if response.status < 400:\n                return True, f\"HTTP {response.status}\"\n            return False, f\"HTTP {response.status}\"\n    except urllib.error.HTTPError as exc:\n        return False, f\"HTTP {exc.code}\"\n    except urllib.error.URLError as exc:\n        reason = getattr(exc, \"reason\", exc)\n        return False, f\"{type(reason).__name__}\"\n    except TimeoutError:\n        return False, \"timeout\"\n\n\ndef candidates(env: Mapping[str, str]) -> list[Candidate]:\n    deepseek_base = env.get(\"DEEPSEEK_BASE_URL\") or \"https://api.deepseek.com\"\n    glm_base = env.get(\"GLM_BASE_URL\") or \"https://open.bigmodel.cn/api/paas/v4\"\n    qwen_base = env.get(\"QWEN_BASE_URL\") or (\n        \"https://dashscope.aliyuncs.com/compatible-mode/v1\"\n    )\n    litellm_base = env.get(\"LITELLM_BASE_URL\") or (\n        \"http://llm-proxy.eval.all-hands.dev/v1\"\n    )\n    github_base = \"https://models.github.ai/inference\"\n\n    return [\n        Candidate(\n            name=\"deepseek\",\n            key_env=\"DEEPSEEK_API_KEY\",\n            base_url=deepseek_base,\n            probe_model=env.get(\"DEEPSEEK_FLASH_MODEL\") or \"deepseek-v4-flash\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"deepseek/deepseek-v4-flash\",\n            judge_model=\"openai/deepseek-v4-flash\",\n            exports={\n                \"DEEPSEEK_API_KEY\": env.get(\"DEEPSEEK_API_KEY\", \"\"),\n                \"DEEPSEEK_BASE_URL\": deepseek_base,\n                \"OPENAI_API_KEY\": env.get(\"DEEPSEEK_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": deepseek_base,\n            },\n        ),\n        Candidate(\n            name=\"glm\",\n            key_env=\"GLM_API_KEY\",\n            base_url=glm_base,\n            probe_model=env.get(\"GLM_MODEL\") or \"glm-5.1\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"glm/glm-5.1\",\n            judge_model=\"openai/glm-5.1\",\n            exports={\n                \"GLM_API_KEY\": env.get(\"GLM_API_KEY\", \"\"),\n                \"GLM_BASE_URL\": glm_base,\n                \"OPENAI_API_KEY\": env.get(\"GLM_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": glm_base,\n            },\n        ),\n        Candidate(\n            name=\"qwen-dashscope\",\n            key_env=\"QWEN_API_KEY\",\n            base_url=qwen_base,\n            probe_model=env.get(\"QWEN_MAX_PREVIEW_MODEL\") or \"qwen3.6-max-preview\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"qwen-dashscope/qwen3.6-max-preview\",\n            judge_model=\"openai/qwen3.6-max-preview\",\n            exports={\n                \"QWEN_API_KEY\": env.get(\"QWEN_API_KEY\", \"\"),\n                \"QWEN_BASE_URL\": qwen_base,\n                \"OPENAI_API_KEY\": env.get(\"QWEN_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": qwen_base,\n            },\n        ),\n        Candidate(\n            name=\"litellm-proxy\",\n            key_env=\"LITELLM_API_KEY\",\n            base_url=litellm_base,\n            probe_model=env.get(\"LITELLM_MODEL\") or \"gpt-4.1-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=f\"litellm/{env.get('LITELLM_MODEL') or 'gpt-4.1-mini'}\",\n            judge_model=f\"openai/{env.get('LITELLM_MODEL') or 'gpt-4.1-mini'}\",\n            exports={\n                \"LITELLM_API_KEY\": env.get(\"LITELLM_API_KEY\", \"\"),\n                \"LITELLM_BASE_URL\": litellm_base,\n                \"OPENAI_API_KEY\": env.get(\"LITELLM_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": litellm_base,\n            },\n        ),\n        Candidate(\n            name=\"openai\",\n            key_env=\"OPENAI_API_KEY\",\n            base_url=env.get(\"OPENAI_BASE_URL\") or \"https://api.openai.com/v1\",\n            probe_model=env.get(\"OPENAI_MODEL\") or \"gpt-4o-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=f\"openai/{env.get('OPENAI_MODEL') or 'gpt-4o-mini'}\",\n            judge_model=f\"openai/{env.get('OPENAI_MODEL') or 'gpt-4o-mini'}\",\n            exports={\n                \"OPENAI_API_KEY\": env.get(\"OPENAI_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": env.get(\"OPENAI_BASE_URL\")\n                or \"https://api.openai.com/v1\",\n            },\n        ),\n        Candidate(\n            name=\"github-models\",\n            key_env=\"GITHUB_MODELS_TOKEN\",\n            base_url=github_base,\n            probe_model=env.get(\"GITHUB_MODELS_MODEL\") or \"openai/gpt-4.1-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"github-models/openai/gpt-4.1-mini\",\n            judge_model=\"openai/openai/gpt-4.1-mini\",\n            exports={\n                \"GITHUB_TOKEN\": env.get(\"GITHUB_MODELS_TOKEN\", \"\"),\n                \"OPENAI_API_KEY\": env.get(\"GITHUB_MODELS_TOKEN\", \"\"),\n                \"OPENAI_BASE_URL\": github_base,\n            },\n        ),\n    ]\n\n\ndef select_candidate(\n    pool: list[Candidate],\n    env: Mapping[str, str],\n    probe: ProbeFunc = probe_candidate,\n) -> tuple[Candidate | None, list[tuple[str, str]]]:\n    attempts: list[tuple[str, str]] = []\n    for candidate in pool:\n        ok, reason = probe(candidate, env)\n        attempts.append((candidate.name, reason))\n        print(f\"Provider candidate {candidate.name}: {reason}\")\n        if ok:\n            return candidate, attempts\n    return None, attempts\n\n\ndef main() -> int:\n    env = os.environ\n    selected, attempts = select_candidate(candidates(env), env)\n    if selected is None:\n        summary = \", \".join(f\"{name}={reason}\" for name, reason in attempts)\n        print(f\"::error::No usable integration LLM provider found ({summary})\")\n        return 1\n\n    exports = {\n        \"BENCHFLOW_INTEGRATION_PROVIDER\": selected.name,\n        \"BENCHFLOW_INTEGRATION_AGENT\": selected.rollout_agent,\n        \"BENCHFLOW_INTEGRATION_MODEL\": selected.rollout_model,\n        \"BENCHFLOW_JUDGE_MODEL\": selected.judge_model,\n        **selected.exports,\n    }\n    _append_github_env(exports, env)\n    print(\n        \"Selected integration provider \"\n        f\"{selected.name}: agent={selected.rollout_agent}, \"\n        f\"model={selected.rollout_model}, judge={selected.judge_model}\"\n    )\n    return 0\n\n\nif __name__ == \"__main__\":\n    sys.exit(main())\n"}, "66448e07a2950fa92cba19919cb37b367e5a9abf": {"t": 1785749199.1144943, "n": 8358, "h": "1868a983378a1658", "c": "#!/usr/bin/env python3\n\"\"\"Select a live model provider for the integration-eval workflow.\n\nThe workflow needs a real LLM for both the rollout and the agent-as-judge\ngate. Repo secrets can drift independently from code, so this script probes the\nconfigured candidates with a tiny OpenAI-compatible request and exports the\nfirst usable one to ``GITHUB_ENV``. It prints only candidate names and HTTP\nstatus classes, never secret values or response bodies.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nimport ssl\nimport sys\nimport urllib.error\nimport urllib.request\nfrom collections.abc import Callable, Mapping\nfrom dataclasses import dataclass\nfrom pathlib import Path\n\nProbeFunc = Callable[[\"Candidate\", Mapping[str, str]], tuple[bool, str]]\n\n\n@dataclass(frozen=True)\nclass Candidate:\n    name: str\n    key_env: str\n    base_url: str\n    probe_model: str\n    rollout_agent: str\n    rollout_model: str\n    judge_model: str\n    exports: Mapping[str, str]\n    insecure_tls: bool = False\n\n\ndef _append_github_env(values: Mapping[str, str], env: Mapping[str, str]) -> None:\n    path = env.get(\"GITHUB_ENV\")\n    if not path:\n        return\n    marker = \"__BENCHFLOW_ENV__\"\n    with Path(path).open(\"a\", encoding=\"utf-8\") as fh:\n        for key, value in values.items():\n            fh.write(f\"{key}<<{marker}\\n{value}\\n{marker}\\n\")\n\n\ndef _chat_completions_url(base_url: str) -> str:\n    return f\"{base_url.rstrip('/')}/chat/completions\"\n\n\ndef probe_candidate(candidate: Candidate, env: Mapping[str, str]) -> tuple[bool, str]:\n    key = env.get(candidate.key_env, \"\").strip()\n    if not key:\n        return False, f\"missing {candidate.key_env}\"\n\n    body = json.dumps(\n        {\n            \"model\": candidate.probe_model,\n            \"messages\": [{\"role\": \"user\", \"content\": \"Return OK only.\"}],\n            \"max_tokens\": 8,\n        }\n    ).encode(\"utf-8\")\n    request = urllib.request.Request(\n        _chat_completions_url(candidate.base_url),\n        data=body,\n        headers={\n            \"Authorization\": f\"Bearer {key}\",\n            \"Content-Type\": \"application/json\",\n        },\n        method=\"POST\",\n    )\n    context = None\n    if candidate.insecure_tls:\n        context = ssl._create_unverified_context()\n    try:\n        with urllib.request.urlopen(request, timeout=25, context=context) as response:\n            if response.status < 400:\n                return True, f\"HTTP {response.status}\"\n            return False, f\"HTTP {response.status}\"\n    except urllib.error.HTTPError as exc:\n        return False, f\"HTTP {exc.code}\"\n    except urllib.error.URLError as exc:\n        reason = getattr(exc, \"reason\", exc)\n        return False, f\"{type(reason).__name__}\"\n    except TimeoutError:\n        return False, \"timeout\"\n\n\ndef candidates(env: Mapping[str, str]) -> list[Candidate]:\n    deepseek_base = env.get(\"DEEPSEEK_BASE_URL\") or \"https://api.deepseek.com\"\n    glm_base = env.get(\"GLM_BASE_URL\") or \"https://open.bigmodel.cn/api/paas/v4\"\n    qwen_base = env.get(\"QWEN_BASE_URL\") or (\n        \"https://dashscope.aliyuncs.com/compatible-mode/v1\"\n    )\n    litellm_base = env.get(\"LITELLM_BASE_URL\") or (\n        \"http://llm-proxy.eval.all-hands.dev/v1\"\n    )\n    github_base = \"https://models.github.ai/inference\"\n\n    return [\n        Candidate(\n            name=\"deepseek\",\n            key_env=\"DEEPSEEK_API_KEY\",\n            base_url=deepseek_base,\n            probe_model=env.get(\"DEEPSEEK_FLASH_MODEL\") or \"deepseek-v4-flash\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"deepseek/deepseek-v4-flash\",\n            judge_model=\"openai/deepseek-v4-flash\",\n            exports={\n                \"DEEPSEEK_API_KEY\": env.get(\"DEEPSEEK_API_KEY\", \"\"),\n                \"DEEPSEEK_BASE_URL\": deepseek_base,\n                \"OPENAI_API_KEY\": env.get(\"DEEPSEEK_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": deepseek_base,\n            },\n        ),\n        Candidate(\n            name=\"glm\",\n            key_env=\"GLM_API_KEY\",\n            base_url=glm_base,\n            probe_model=env.get(\"GLM_MODEL\") or \"glm-5.1\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"glm/glm-5.1\",\n            judge_model=\"openai/glm-5.1\",\n            exports={\n                \"GLM_API_KEY\": env.get(\"GLM_API_KEY\", \"\"),\n                \"GLM_BASE_URL\": glm_base,\n                \"OPENAI_API_KEY\": env.get(\"GLM_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": glm_base,\n            },\n        ),\n        Candidate(\n            name=\"qwen-dashscope\",\n            key_env=\"QWEN_API_KEY\",\n            base_url=qwen_base,\n            probe_model=env.get(\"QWEN_MAX_PREVIEW_MODEL\") or \"qwen3.6-max-preview\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"qwen-dashscope/qwen3.6-max-preview\",\n            judge_model=\"openai/qwen3.6-max-preview\",\n            exports={\n                \"QWEN_API_KEY\": env.get(\"QWEN_API_KEY\", \"\"),\n                \"QWEN_BASE_URL\": qwen_base,\n                \"OPENAI_API_KEY\": env.get(\"QWEN_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": qwen_base,\n            },\n        ),\n        Candidate(\n            name=\"litellm-proxy\",\n            key_env=\"LITELLM_API_KEY\",\n            base_url=litellm_base,\n            probe_model=env.get(\"LITELLM_MODEL\") or \"gpt-4.1-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=f\"litellm/{env.get('LITELLM_MODEL') or 'gpt-4.1-mini'}\",\n            judge_model=f\"openai/{env.get('LITELLM_MODEL') or 'gpt-4.1-mini'}\",\n            exports={\n                \"LITELLM_API_KEY\": env.get(\"LITELLM_API_KEY\", \"\"),\n                \"LITELLM_BASE_URL\": litellm_base,\n                \"OPENAI_API_KEY\": env.get(\"LITELLM_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": litellm_base,\n            },\n        ),\n        Candidate(\n            name=\"openai\",\n            key_env=\"OPENAI_API_KEY\",\n            base_url=env.get(\"OPENAI_BASE_URL\") or \"https://api.openai.com/v1\",\n            probe_model=env.get(\"OPENAI_MODEL\") or \"gpt-4o-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=f\"openai/{env.get('OPENAI_MODEL') or 'gpt-4o-mini'}\",\n            judge_model=f\"openai/{env.get('OPENAI_MODEL') or 'gpt-4o-mini'}\",\n            exports={\n                \"OPENAI_API_KEY\": env.get(\"OPENAI_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": env.get(\"OPENAI_BASE_URL\")\n                or \"https://api.openai.com/v1\",\n            },\n        ),\n        Candidate(\n            name=\"github-models\",\n            key_env=\"GITHUB_MODELS_TOKEN\",\n            base_url=github_base,\n            probe_model=env.get(\"GITHUB_MODELS_MODEL\") or \"openai/gpt-4.1-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"github-models/openai/gpt-4.1-mini\",\n            judge_model=\"openai/openai/gpt-4.1-mini\",\n            exports={\n                \"GITHUB_TOKEN\": env.get(\"GITHUB_MODELS_TOKEN\", \"\"),\n                \"OPENAI_API_KEY\": env.get(\"GITHUB_MODELS_TOKEN\", \"\"),\n                \"OPENAI_BASE_URL\": github_base,\n            },\n        ),\n    ]\n\n\ndef select_candidate(\n    pool: list[Candidate],\n    env: Mapping[str, str],\n    probe: ProbeFunc = probe_candidate,\n) -> tuple[Candidate | None, list[tuple[str, str]]]:\n    attempts: list[tuple[str, str]] = []\n    for candidate in pool:\n        ok, reason = probe(candidate, env)\n        attempts.append((candidate.name, reason))\n        print(f\"Provider candidate {candidate.name}: {reason}\")\n        if ok:\n            return candidate, attempts\n    return None, attempts\n\n\ndef main() -> int:\n    env = os.environ\n    selected, attempts = select_candidate(candidates(env), env)\n    if selected is None:\n        summary = \", \".join(f\"{name}={reason}\" for name, reason in attempts)\n        print(f\"::error::No usable integration LLM provider found ({summary})\")\n        return 1\n\n    exports = {\n        \"BENCHFLOW_INTEGRATION_PROVIDER\": selected.name,\n        \"BENCHFLOW_INTEGRATION_AGENT\": selected.rollout_agent,\n        \"BENCHFLOW_INTEGRATION_MODEL\": selected.rollout_model,\n        \"BENCHFLOW_JUDGE_MODEL\": selected.judge_model,\n        **selected.exports,\n    }\n    _append_github_env(exports, env)\n    print(\n        \"Selected integration provider \"\n        f\"{selected.name}: agent={selected.rollout_agent}, \"\n        f\"model={selected.rollout_model}, judge={selected.judge_model}\"\n    )\n    return 0\n\n\nif __name__ == \"__main__\":\n    sys.exit(main())\n"}, "2631b1d47436fbaa6017956c2957e7d4147b3cae": {"t": 1786068534.9841871, "n": 8358, "h": "1868a983378a1658", "c": "#!/usr/bin/env python3\n\"\"\"Select a live model provider for the integration-eval workflow.\n\nThe workflow needs a real LLM for both the rollout and the agent-as-judge\ngate. Repo secrets can drift independently from code, so this script probes the\nconfigured candidates with a tiny OpenAI-compatible request and exports the\nfirst usable one to ``GITHUB_ENV``. It prints only candidate names and HTTP\nstatus classes, never secret values or response bodies.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nimport ssl\nimport sys\nimport urllib.error\nimport urllib.request\nfrom collections.abc import Callable, Mapping\nfrom dataclasses import dataclass\nfrom pathlib import Path\n\nProbeFunc = Callable[[\"Candidate\", Mapping[str, str]], tuple[bool, str]]\n\n\n@dataclass(frozen=True)\nclass Candidate:\n    name: str\n    key_env: str\n    base_url: str\n    probe_model: str\n    rollout_agent: str\n    rollout_model: str\n    judge_model: str\n    exports: Mapping[str, str]\n    insecure_tls: bool = False\n\n\ndef _append_github_env(values: Mapping[str, str], env: Mapping[str, str]) -> None:\n    path = env.get(\"GITHUB_ENV\")\n    if not path:\n        return\n    marker = \"__BENCHFLOW_ENV__\"\n    with Path(path).open(\"a\", encoding=\"utf-8\") as fh:\n        for key, value in values.items():\n            fh.write(f\"{key}<<{marker}\\n{value}\\n{marker}\\n\")\n\n\ndef _chat_completions_url(base_url: str) -> str:\n    return f\"{base_url.rstrip('/')}/chat/completions\"\n\n\ndef probe_candidate(candidate: Candidate, env: Mapping[str, str]) -> tuple[bool, str]:\n    key = env.get(candidate.key_env, \"\").strip()\n    if not key:\n        return False, f\"missing {candidate.key_env}\"\n\n    body = json.dumps(\n        {\n            \"model\": candidate.probe_model,\n            \"messages\": [{\"role\": \"user\", \"content\": \"Return OK only.\"}],\n            \"max_tokens\": 8,\n        }\n    ).encode(\"utf-8\")\n    request = urllib.request.Request(\n        _chat_completions_url(candidate.base_url),\n        data=body,\n        headers={\n            \"Authorization\": f\"Bearer {key}\",\n            \"Content-Type\": \"application/json\",\n        },\n        method=\"POST\",\n    )\n    context = None\n    if candidate.insecure_tls:\n        context = ssl._create_unverified_context()\n    try:\n        with urllib.request.urlopen(request, timeout=25, context=context) as response:\n            if response.status < 400:\n                return True, f\"HTTP {response.status}\"\n            return False, f\"HTTP {response.status}\"\n    except urllib.error.HTTPError as exc:\n        return False, f\"HTTP {exc.code}\"\n    except urllib.error.URLError as exc:\n        reason = getattr(exc, \"reason\", exc)\n        return False, f\"{type(reason).__name__}\"\n    except TimeoutError:\n        return False, \"timeout\"\n\n\ndef candidates(env: Mapping[str, str]) -> list[Candidate]:\n    deepseek_base = env.get(\"DEEPSEEK_BASE_URL\") or \"https://api.deepseek.com\"\n    glm_base = env.get(\"GLM_BASE_URL\") or \"https://open.bigmodel.cn/api/paas/v4\"\n    qwen_base = env.get(\"QWEN_BASE_URL\") or (\n        \"https://dashscope.aliyuncs.com/compatible-mode/v1\"\n    )\n    litellm_base = env.get(\"LITELLM_BASE_URL\") or (\n        \"http://llm-proxy.eval.all-hands.dev/v1\"\n    )\n    github_base = \"https://models.github.ai/inference\"\n\n    return [\n        Candidate(\n            name=\"deepseek\",\n            key_env=\"DEEPSEEK_API_KEY\",\n            base_url=deepseek_base,\n            probe_model=env.get(\"DEEPSEEK_FLASH_MODEL\") or \"deepseek-v4-flash\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"deepseek/deepseek-v4-flash\",\n            judge_model=\"openai/deepseek-v4-flash\",\n            exports={\n                \"DEEPSEEK_API_KEY\": env.get(\"DEEPSEEK_API_KEY\", \"\"),\n                \"DEEPSEEK_BASE_URL\": deepseek_base,\n                \"OPENAI_API_KEY\": env.get(\"DEEPSEEK_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": deepseek_base,\n            },\n        ),\n        Candidate(\n            name=\"glm\",\n            key_env=\"GLM_API_KEY\",\n            base_url=glm_base,\n            probe_model=env.get(\"GLM_MODEL\") or \"glm-5.1\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"glm/glm-5.1\",\n            judge_model=\"openai/glm-5.1\",\n            exports={\n                \"GLM_API_KEY\": env.get(\"GLM_API_KEY\", \"\"),\n                \"GLM_BASE_URL\": glm_base,\n                \"OPENAI_API_KEY\": env.get(\"GLM_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": glm_base,\n            },\n        ),\n        Candidate(\n            name=\"qwen-dashscope\",\n            key_env=\"QWEN_API_KEY\",\n            base_url=qwen_base,\n            probe_model=env.get(\"QWEN_MAX_PREVIEW_MODEL\") or \"qwen3.6-max-preview\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"qwen-dashscope/qwen3.6-max-preview\",\n            judge_model=\"openai/qwen3.6-max-preview\",\n            exports={\n                \"QWEN_API_KEY\": env.get(\"QWEN_API_KEY\", \"\"),\n                \"QWEN_BASE_URL\": qwen_base,\n                \"OPENAI_API_KEY\": env.get(\"QWEN_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": qwen_base,\n            },\n        ),\n        Candidate(\n            name=\"litellm-proxy\",\n            key_env=\"LITELLM_API_KEY\",\n            base_url=litellm_base,\n            probe_model=env.get(\"LITELLM_MODEL\") or \"gpt-4.1-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=f\"litellm/{env.get('LITELLM_MODEL') or 'gpt-4.1-mini'}\",\n            judge_model=f\"openai/{env.get('LITELLM_MODEL') or 'gpt-4.1-mini'}\",\n            exports={\n                \"LITELLM_API_KEY\": env.get(\"LITELLM_API_KEY\", \"\"),\n                \"LITELLM_BASE_URL\": litellm_base,\n                \"OPENAI_API_KEY\": env.get(\"LITELLM_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": litellm_base,\n            },\n        ),\n        Candidate(\n            name=\"openai\",\n            key_env=\"OPENAI_API_KEY\",\n            base_url=env.get(\"OPENAI_BASE_URL\") or \"https://api.openai.com/v1\",\n            probe_model=env.get(\"OPENAI_MODEL\") or \"gpt-4o-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=f\"openai/{env.get('OPENAI_MODEL') or 'gpt-4o-mini'}\",\n            judge_model=f\"openai/{env.get('OPENAI_MODEL') or 'gpt-4o-mini'}\",\n            exports={\n                \"OPENAI_API_KEY\": env.get(\"OPENAI_API_KEY\", \"\"),\n                \"OPENAI_BASE_URL\": env.get(\"OPENAI_BASE_URL\")\n                or \"https://api.openai.com/v1\",\n            },\n        ),\n        Candidate(\n            name=\"github-models\",\n            key_env=\"GITHUB_MODELS_TOKEN\",\n            base_url=github_base,\n            probe_model=env.get(\"GITHUB_MODELS_MODEL\") or \"openai/gpt-4.1-mini\",\n            rollout_agent=\"openhands\",\n            rollout_model=\"github-models/openai/gpt-4.1-mini\",\n            judge_model=\"openai/openai/gpt-4.1-mini\",\n            exports={\n                \"GITHUB_TOKEN\": env.get(\"GITHUB_MODELS_TOKEN\", \"\"),\n                \"OPENAI_API_KEY\": env.get(\"GITHUB_MODELS_TOKEN\", \"\"),\n                \"OPENAI_BASE_URL\": github_base,\n            },\n        ),\n    ]\n\n\ndef select_candidate(\n    pool: list[Candidate],\n    env: Mapping[str, str],\n    probe: ProbeFunc = probe_candidate,\n) -> tuple[Candidate | None, list[tuple[str, str]]]:\n    attempts: list[tuple[str, str]] = []\n    for candidate in pool:\n        ok, reason = probe(candidate, env)\n        attempts.append((candidate.name, reason))\n        print(f\"Provider candidate {candidate.name}: {reason}\")\n        if ok:\n            return candidate, attempts\n    return None, attempts\n\n\ndef main() -> int:\n    env = os.environ\n    selected, attempts = select_candidate(candidates(env), env)\n    if selected is None:\n        summary = \", \".join(f\"{name}={reason}\" for name, reason in attempts)\n        print(f\"::error::No usable integration LLM provider found ({summary})\")\n        return 1\n\n    exports = {\n        \"BENCHFLOW_INTEGRATION_PROVIDER\": selected.name,\n        \"BENCHFLOW_INTEGRATION_AGENT\": selected.rollout_agent,\n        \"BENCHFLOW_INTEGRATION_MODEL\": selected.rollout_model,\n        \"BENCHFLOW_JUDGE_MODEL\": selected.judge_model,\n        **selected.exports,\n    }\n    _append_github_env(exports, env)\n    print(\n        \"Selected integration provider \"\n        f\"{selected.name}: agent={selected.rollout_agent}, \"\n        f\"model={selected.rollout_model}, judge={selected.judge_model}\"\n    )\n    return 0\n\n\nif __name__ == \"__main__\":\n    sys.exit(main())\n"}}, ".github/integration/ADMIN_SETUP.md": {"cf656179f3bc30b2887b744cf71b69b5e5ffa5ff": {"t": 1785747012.8315158, "n": 7638, "h": "812657b56f33f0d2", "c": "# Integration CI — admin setup (protected env, branch protection, secrets)\n\nOne-time GitHub configuration the L0–L3 integration workflows depend on. None of\nthis is enforceable from a workflow file; a repo admin must set it in the GitHub\nUI / API. Workflows:\n\n| Level | File | Trigger | Required? |\n|------|------|---------|-----------|\n| L0 | `.github/workflows/test.yml` | push / PR / merge_group | yes (every PR) |\n| L1 | `.github/workflows/integration-light.yml` | PR (opened/sync/reopen/ready) + dispatch | yes (every PR) |\n| L2 | `.github/workflows/integration-scope.yml` | PR (labeled `integration:medium`, …) + dispatch | yes (real work only on label) |\n| L3 | `.github/workflows/integration-final-review.yml` | `workflow_dispatch` only | gated by protected env approval |\n\n## 1. Branch protection (required status checks)\n\nOn the protected `main` branch, mark these checks **required**:\n\n- `test` (L0 unit/lint/type lane)\n- L1 job check from `integration-light` — the `detect-scope` + `rollout-smoke`\n  jobs. Because `detect-scope` reports a **green no-op** (`should_run=false`)\n  when a diff needs no rollout, this check can be required **unconditionally**\n  without blocking docs-only PRs.\n- L2 job check from `integration-scope` — likewise its `detect-scope` is a cheap\n  green no-op when the `integration:medium` label is absent, so it is safe to\n  require unconditionally.\n\n> Required checks must **always run** (no `on: paths` filter) or a PR that\n> doesn't touch the filtered paths would wait forever on a check that never\n> starts. That is why every level's first job is the secret-free `detect-scope`\n> no-op rather than a path filter.\n\nL3 is **not** a required status check. It is enforced by the protected\nenvironment approval below, plus the `integration-final-review` check-run that\nthe L3 review job publishes on the PR head SHA.\n\n## 2. Protected environments\n\nUse the existing **`pypi-internal-preview`** environment for L1/L2/L3 — **no\nseparate protected environment is created** (per the agreed simplification, L3's\ngolden truth is the HuggingFace leaderboard `main` runs vs the latest benchflow\nmain, not a human approval gate).\n\n### `pypi-internal-preview` (L1 + L2 + L3)\n\n- Holds the provider/Daytona/reviewer secrets (see §3).\n- No required reviewers — L1/L2 run automatically on internal PRs; L3 is a manual\n  `workflow_dispatch`.\n- L1 attaches **only low-value** provider keys (DEEPSEEK / GEMINI /\n  GITHUB_MODELS). L2/L3 attach **all** keys per the agreed Q6 residual. The L3\n  codex reviewer authenticates with the **existing repo `OPENAI_API_KEY`** — no\n  separate secret to create.\n- *Optional, later:* to add a hard human gate, create a protected environment\n  with required reviewers and point the L3 `run-matrix` / `review-pack` jobs at it.\n\n## 3. Secrets\n\nSet on each environment (not as bare repo secrets) so they rotate without\nrepo-admin Actions access. Most `*_BASE_URL` are optional overrides — but\n`DEEPSEEK_BASE_URL` is **required** (see below).\n\nProvider (rollout + judge):\n`DEEPSEEK_API_KEY`, `DEEPSEEK_BASE_URL`, `GLM_API_KEY`, `GLM_BASE_URL`,\n`QWEN_API_KEY`, `QWEN_BASE_URL`, `GEMINI_API_KEY`, `OPENAI_API_KEY`,\n`LITELLM_API_KEY` (or `BF_TOKEN`), `LITELLM_BASE_URL`, `GITHUB_MODELS_TOKEN`.\n\nPer the agent model policy (see\n[`../../docs/integration-tiers.md`](../../docs/integration-tiers.md) §3.3), these\nspecific keys are load-bearing:\n\n- **`DEEPSEEK_API_KEY` + `DEEPSEEK_BASE_URL` — both required; gate rollout AND\n  grading.** The 5 open agents (`openhands`, `openclaw`, `opencode`, `pi-acp`,\n  `mimo`) run on `deepseek/deepseek-v4-flash` through the LiteLLM usage proxy; the\n  `deepseek` provider declares `url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"}`\n  (`src/benchflow/agents/providers.py`), so `DEEPSEEK_BASE_URL` is **not** an\n  optional override. The **agent-judge also runs on DeepSeek-v4 only**\n  (`select_integration_provider.py` exports\n  `BENCHFLOW_JUDGE_MODEL=openai/deepseek-v4-flash`, routed via the OpenAI-\n  compatible surface with the DeepSeek key/base_url). The judge is NOT a matrix\n  cell, so the credential filter does **not** protect it — a missing\n  `DEEPSEEK_*` fails grading on **every** cell.\n- **`GEMINI_API_KEY` — set but NOT load-bearing.** Nothing on the default path\n  consumes it: the judge runs on DeepSeek-v4 and the `gemini` /\n  `harvey-lab-harness` rollout agents were dropped from the roster. It is\n  provisioned on the environment as an available fallback only (the judge's\n  failover order in `select_integration_provider.py` is DeepSeek → GLM → Qwen →\n  LiteLLM → OpenAI → GitHub Models; gemini is not a judge candidate).\n- **`OPENAI_API_KEY`** — `codex-acp` rollout model (`gpt-5.4-nano`) **and** the L3\n  Codex equivalence reviewer. `codex-acp` is a GATED native (see below).\n- **`AWS_BEARER_TOKEN_BEDROCK` (+ `AWS_REGION`) — `claude-agent-acp` credential,\n  present in `pypi-internal-preview`.** `claude-agent-acp` routes through\n  Bedrock's `anthropic-messages` surface\n  (`aws-bedrock/us.anthropic.claude-haiku-4-5-20251001`), which requires\n  `AWS_BEARER_TOKEN_BEDROCK` and `AWS_REGION`\n  (`src/benchflow/providers/litellm_config.py`). Both keys are set on the\n  environment. If they are ever absent, credential-aware emission\n  ([`../scripts/filter_credentialed_cells.py`](../scripts/filter_credentialed_cells.py))\n  **drops** the `claude` cells as a documented skip (not a red slot).\n\n> **Roster & gating (current policy).** The broad fan is **DeepSeek-only**: the 5\n> DeepSeek agents (`openhands`, `pi-acp`, `openclaw`, `opencode`, `mimo`) run in\n> every roster lane. The 2 **gated natives** — `claude-agent-acp` (Bedrock) and\n> `codex-acp` (OpenAI) — cannot use DeepSeek and are blocked from the default\n> fan; they run **only via affected-agent** (a PR touching their own adapter), so\n> a default run needs only `DEEPSEEK_*` (rollout **and** judge) + `DAYTONA_API_KEY`.\n> `gemini` and `harvey-lab-harness` were removed from the roster.\n\nSandbox: `DAYTONA_API_KEY` (L2/L3 only; L1 is docker-only).\n\nCodex reviewer (L3, required): the **existing repo `OPENAI_API_KEY`** — durable\nand revocable, unlike a personal ChatGPT OAuth blob. `codex_review.py` writes it\nas an apikey `auth.json` at the codex config path (`$CODEX_HOME/auth.json`,\ndefault `~/.codex/auth.json`) before invoking `codex exec`. Auth precedence:\n`OPENAI_API_KEY` / `CODEX_API_KEY`, then an optional `CODEX_AUTH_JSON` blob, then\na pre-existing on-host `auth.json`. If none resolve, L3 fails closed to\n**`not mergeable (codex unavailable)`**.\n\n## 4. Labels\n\n- `integration:medium` — opens the L2 lane on a PR (its `detect-scope` only does\n  real work when this label is present).\n- `integration:deep` — signals an L2/L3-class change; L1's `rollout-smoke` steps\n  aside when present so the lanes don't double-spend provider credits.\n\n## 5. Security invariants (do not weaken)\n\n- All workflows are plain `pull_request` / `workflow_dispatch`, never\n  `pull_request_target`. Fork PRs receive no secrets.\n- `detect-scope`, `plan`, and `review-pack` check out the planner / grader /\n  harness / review skill from **trusted `origin/main`** (sparse), never the PR\n  head. Only `src/benchflow` (the orchestrator code-under-test) is overlaid from\n  the PR head SHA.\n- The L3 final verdict is `worst(deterministic, codex)`: codex can only make the\n  verdict stricter, never upgrade a deterministic `not mergeable`.\n- Actions are SHA-pinned. `actions/download-artifact`\n  (`d3f86a106a0bac45b974a628896c90dbdf5c8093`, v4.3.0) and\n  `actions/github-script` (`60a0d83039c74a4aee543508d2ffcb1c3799cdea`, v7.0.1)\n  are new surface relative to L0/L1's existing pins — re-verify on bump.\n"}, "e1060f6cc3eca6226f60971be9b2dc340763c15a": {"t": 1785749196.142417, "n": 7638, "h": "812657b56f33f0d2", "c": "# Integration CI — admin setup (protected env, branch protection, secrets)\n\nOne-time GitHub configuration the L0–L3 integration workflows depend on. None of\nthis is enforceable from a workflow file; a repo admin must set it in the GitHub\nUI / API. Workflows:\n\n| Level | File | Trigger | Required? |\n|------|------|---------|-----------|\n| L0 | `.github/workflows/test.yml` | push / PR / merge_group | yes (every PR) |\n| L1 | `.github/workflows/integration-light.yml` | PR (opened/sync/reopen/ready) + dispatch | yes (every PR) |\n| L2 | `.github/workflows/integration-scope.yml` | PR (labeled `integration:medium`, …) + dispatch | yes (real work only on label) |\n| L3 | `.github/workflows/integration-final-review.yml` | `workflow_dispatch` only | gated by protected env approval |\n\n## 1. Branch protection (required status checks)\n\nOn the protected `main` branch, mark these checks **required**:\n\n- `test` (L0 unit/lint/type lane)\n- L1 job check from `integration-light` — the `detect-scope` + `rollout-smoke`\n  jobs. Because `detect-scope` reports a **green no-op** (`should_run=false`)\n  when a diff needs no rollout, this check can be required **unconditionally**\n  without blocking docs-only PRs.\n- L2 job check from `integration-scope` — likewise its `detect-scope` is a cheap\n  green no-op when the `integration:medium` label is absent, so it is safe to\n  require unconditionally.\n\n> Required checks must **always run** (no `on: paths` filter) or a PR that\n> doesn't touch the filtered paths would wait forever on a check that never\n> starts. That is why every level's first job is the secret-free `detect-scope`\n> no-op rather than a path filter.\n\nL3 is **not** a required status check. It is enforced by the protected\nenvironment approval below, plus the `integration-final-review` check-run that\nthe L3 review job publishes on the PR head SHA.\n\n## 2. Protected environments\n\nUse the existing **`pypi-internal-preview`** environment for L1/L2/L3 — **no\nseparate protected environment is created** (per the agreed simplification, L3's\ngolden truth is the HuggingFace leaderboard `main` runs vs the latest benchflow\nmain, not a human approval gate).\n\n### `pypi-internal-preview` (L1 + L2 + L3)\n\n- Holds the provider/Daytona/reviewer secrets (see §3).\n- No required reviewers — L1/L2 run automatically on internal PRs; L3 is a manual\n  `workflow_dispatch`.\n- L1 attaches **only low-value** provider keys (DEEPSEEK / GEMINI /\n  GITHUB_MODELS). L2/L3 attach **all** keys per the agreed Q6 residual. The L3\n  codex reviewer authenticates with the **existing repo `OPENAI_API_KEY`** — no\n  separate secret to create.\n- *Optional, later:* to add a hard human gate, create a protected environment\n  with required reviewers and point the L3 `run-matrix` / `review-pack` jobs at it.\n\n## 3. Secrets\n\nSet on each environment (not as bare repo secrets) so they rotate without\nrepo-admin Actions access. Most `*_BASE_URL` are optional overrides — but\n`DEEPSEEK_BASE_URL` is **required** (see below).\n\nProvider (rollout + judge):\n`DEEPSEEK_API_KEY`, `DEEPSEEK_BASE_URL`, `GLM_API_KEY`, `GLM_BASE_URL`,\n`QWEN_API_KEY`, `QWEN_BASE_URL`, `GEMINI_API_KEY`, `OPENAI_API_KEY`,\n`LITELLM_API_KEY` (or `BF_TOKEN`), `LITELLM_BASE_URL`, `GITHUB_MODELS_TOKEN`.\n\nPer the agent model policy (see\n[`../../docs/integration-tiers.md`](../../docs/integration-tiers.md) §3.3), these\nspecific keys are load-bearing:\n\n- **`DEEPSEEK_API_KEY` + `DEEPSEEK_BASE_URL` — both required; gate rollout AND\n  grading.** The 5 open agents (`openhands`, `openclaw`, `opencode`, `pi-acp`,\n  `mimo`) run on `deepseek/deepseek-v4-flash` through the LiteLLM usage proxy; the\n  `deepseek` provider declares `url_params={\"base_url\": \"DEEPSEEK_BASE_URL\"}`\n  (`src/benchflow/agents/providers.py`), so `DEEPSEEK_BASE_URL` is **not** an\n  optional override. The **agent-judge also runs on DeepSeek-v4 only**\n  (`select_integration_provider.py` exports\n  `BENCHFLOW_JUDGE_MODEL=openai/deepseek-v4-flash`, routed via the OpenAI-\n  compatible surface with the DeepSeek key/base_url). The judge is NOT a matrix\n  cell, so the credential filter does **not** protect it — a missing\n  `DEEPSEEK_*` fails grading on **every** cell.\n- **`GEMINI_API_KEY` — set but NOT load-bearing.** Nothing on the default path\n  consumes it: the judge runs on DeepSeek-v4 and the `gemini` /\n  `harvey-lab-harness` rollout agents were dropped from the roster. It is\n  provisioned on the environment as an available fallback only (the judge's\n  failover order in `select_integration_provider.py` is DeepSeek → GLM → Qwen →\n  LiteLLM → OpenAI → GitHub Models; gemini is not a judge candidate).\n- **`OPENAI_API_KEY`** — `codex-acp` rollout model (`gpt-5.4-nano`) **and** the L3\n  Codex equivalence reviewer. `codex-acp` is a GATED native (see below).\n- **`AWS_BEARER_TOKEN_BEDROCK` (+ `AWS_REGION`) — `claude-agent-acp` credential,\n  present in `pypi-internal-preview`.** `claude-agent-acp` routes through\n  Bedrock's `anthropic-messages` surface\n  (`aws-bedrock/us.anthropic.claude-haiku-4-5-20251001`), which requires\n  `AWS_BEARER_TOKEN_BEDROCK` and `AWS_REGION`\n  (`src/benchflow/providers/litellm_config.py`). Both keys are set on the\n  environment. If they are ever absent, credential-aware emission\n  ([`../scripts/filter_credentialed_cells.py`](../scripts/filter_credentialed_cells.py))\n  **drops** the `claude` cells as a documented skip (not a red slot).\n\n> **Roster & gating (current policy).** The broad fan is **DeepSeek-only**: the 5\n> DeepSeek agents (`openhands`, `pi-acp`, `openclaw`, `opencode`, `mimo`) run in\n> every roster lane. The 2 **gated natives** — `claude-agent-acp` (Bedrock) and\n> `codex-acp` (OpenAI) — cannot use DeepSeek and are blocked from the default\n> fan; they run **only via affected-agent** (a PR touching their own adapter), so\n> a default run needs only `DEEPSEEK_*` (rollout **and** judge) + `DAYTONA_API_KEY`.\n> `gemini` and `harvey-lab-harness` were removed from the roster.\n\nSandbox: `DAYTONA_API_KEY` (L2/L3 only; L1 is docker-only).\n\nCodex reviewer (L3, required): the **existing repo `OPENAI_API_KEY`** — durable\nand revocable, unlike a personal ChatGPT OAuth blob. `codex_review.py` writes it\nas an apikey `auth.json` at the codex config path (`$CODEX_HOME/auth.json`,\ndefault `~/.codex/auth.json`) before invoking `codex exec`. Auth precedence:\n`OPENAI_API_KEY` / `CODEX_API_KEY`, then an optional `CODEX_AUTH_JSON` blob, then\na pre-existing on-host `auth.json`. If none resolve, L3 fails closed to\n**`not mergeable (codex unavailable)`**.\n\n## 4. Labels\n\n- `integration:medium` — opens the L2 lane on a PR (its `detect-scope` only does\n  real work when this label is present).\n- `integration:deep` — signals an L2/L3-class change; L1's `rollout-smoke` steps\n  aside when present so the lanes don't double-spend provider credits.\n\n## 5. Security invariants (do not weaken)\n\n- All workflows are plain `pull_request` / `workflow_dispatch`, never\n  `pull_request_target`. Fork PRs receive no secrets.\n- `detect-scope`, `plan`, and `review-pack` check out the planner / grader /\n  harness / review skill from **trusted `origin/main`** (sparse), never the PR\n  head. Only `src/benchflow` (the orchestrator code-under-test) is overlaid from\n  the PR head SHA.\n- The L3 final verdict is `worst(deterministic, codex)`: codex can only make the\n  verdict stricter, never upgrade a deterministic `not mergeable`.\n- Actions are SHA-pinned. `actions/download-artifact`\n  (`d3f86a106a0bac45b974a628896c90dbdf5c8093`, v4.3.0) and\n  `actions/github-script` (`60a0d83039c74a4aee543508d2ffcb1c3799cdea`, v7.0.1)\n  are new surface relative to L0/L1's existing pins — re-verify on bump.\n"}, "2631b1d47436fbaa6017956c2957e7d4147b3cae": {"t": 1786068536.61991, "n": 7835, "h": "00b4f1404c54d346", "c": "# Integration CI — admin setup (protected env, branch protection, secrets)\n\nOne-time GitHub configuration the L0–L3 integration workflows depend on. None of\nthis is enforceable from a workflow file; a repo admin must set it in the GitHub\nUI / API. Workflows:\n\n| Level | File | Trigger | Required? |\n|------|------|---------|-----------|\n| L0 | `.github/workflows/test.yml` | push / PR / merge_group | yes (every PR) |\n| L1 | `.github/workflows/integration-light.yml` | PR + dispatch + successful `test` run on `main` | yes (every PR; also gates previews) |\n| L2 | `.github/workflows/integration-scope.yml` | PR (labeled `integration:medium`, …) + dispatch | yes (real work only on label) |\n| L3 | `.github/workflows/integration-final-review.yml` | `workflow_dispatch` only | gated by protected env approval |\n\n## 1. Branch protection (required status checks)\n\nOn the protected `main` branch, mark these checks **required**:\n\n- `test` (L0 unit/lint/type lane)\n- L1 job check from `integration-light` — the `detect-scope` + `rollout-smoke`\n  jobs. Because `detect-scope` reports a **green no-op** (`should_run=false`)\n  when a diff needs no rollout, this check can be required **unconditionally**\n  without blocking docs-only PRs.\n- L2 job check from `integration-scope` — likewise its `detect-scope` is a cheap\n  green no-op when the `integration:medium` label is absent, so it is safe to\n  require unconditionally.\n\n> Required checks must **always run** (no `on: paths` filter) or a PR that\n> doesn't touch the filtered paths would wait forever on a check that never\n> starts. That is why every level's first job is the secret-free `detect-scope`\n> no-op rather than a path filter.\n\nL3 is **not** a required status check. It is enforced by the protected\nenvironment approval below, plus the `integration-final-review` check-run that\nthe L3 review job publishes on the PR head SHA.\n\n## 2. Protected environments\n\nUse the existing **`pypi-internal-preview`** environment for L1/L2/L3 — **no\nseparate protected environment is created** (per the agreed simplification, L3's\ngolden truth is the HuggingFace leaderboard `main` runs vs the latest benchflow\nmain, not a human approval gate).\n\n### `pypi-internal-preview` (L1 + L2 + L3)\n\n- Holds the provider/Daytona/reviewer secrets (see §3).\n- No required reviewers — L1/L2 run automatically on internal PRs; L3 is a manual\n  `workflow_dispatch`.\n- L1 attaches **only low-value** provider keys (DEEPSEEK / GEMINI /\n  GITHUB_MODELS). L2/L3 attach **all** keys per the agreed Q6 residual. The L3\n  codex reviewer authenticates with the **existing repo `OPENAI_API_KEY`** — no\n  separate secret to create.\n- *Optional, later:* to add a hard human gate, create a protected environment\n  with required reviewers and point the L3 `run-matrix` / `review-pack` jobs at it.\n\n## 3. Secrets\n\nSet on each environment (not as bare repo secrets) so they rotate without\nrepo-admin Actions access. `*_BASE_URL` values are optional when the provider\nhas a built-in public endpoint; set one when routing through a custom gateway.\n\nProvider (rollout + judge):\n`DEEPSEEK_API_KEY`, `DEEPSEEK_BASE_URL`, `GLM_API_KEY`, `GLM_BASE_URL`,\n`QWEN_API_KEY`, `QWEN_BASE_URL`, `GEMINI_API_KEY`, `OPENAI_API_KEY`,\n`LITELLM_API_KEY` (or `BF_TOKEN`), `LITELLM_BASE_URL`, `GITHUB_MODELS_TOKEN`.\n\nPer the agent model policy (see\n[`../../docs/integration-tiers.md`](../../docs/integration-tiers.md) §3.3), these\nspecific keys are load-bearing:\n\n- **`DEEPSEEK_API_KEY` — gates rollout AND grading; `DEEPSEEK_BASE_URL` is an\n  optional override.** The 5 open agents (`openhands`, `openclaw`, `opencode`, `pi-acp`,\n  `mimo`) run on `deepseek/deepseek-v4-flash` through the LiteLLM usage proxy; the\n  provider defaults to DeepSeek's public OpenAI-compatible endpoint and honors\n  `DEEPSEEK_BASE_URL` when set. The **agent-judge also runs on DeepSeek-v4 only**\n  (`select_integration_provider.py` exports\n  `BENCHFLOW_JUDGE_MODEL=openai/deepseek-v4-flash`, routed via the OpenAI-\n  compatible surface with the DeepSeek key/base_url). The judge is NOT a matrix\n  cell, so the credential filter does **not** protect it — a missing\n  `DEEPSEEK_API_KEY` fails grading on **every** cell.\n- **`GEMINI_API_KEY` — set but NOT load-bearing.** Nothing on the default path\n  consumes it: the judge runs on DeepSeek-v4 and the `gemini` /\n  `harvey-lab-harness` rollout agents were dropped from the roster. It is\n  provisioned on the environment as an available fallback only (the judge's\n  failover order in `select_integration_provider.py` is DeepSeek → GLM → Qwen →\n  LiteLLM → OpenAI → GitHub Models; gemini is not a judge candidate).\n- **`OPENAI_API_KEY`** — `codex-acp` rollout model (`gpt-5.4-nano`) **and** the L3\n  Codex equivalence reviewer. `codex-acp` is a GATED native (see below).\n- **`AWS_BEARER_TOKEN_BEDROCK` (+ `AWS_REGION`) — `claude-agent-acp` credential,\n  present in `pypi-internal-preview`.** `claude-agent-acp` routes through\n  Bedrock's `anthropic-messages` surface\n  (`aws-bedrock/us.anthropic.claude-haiku-4-5-20251001`), which requires\n  `AWS_BEARER_TOKEN_BEDROCK` and `AWS_REGION`\n  (`src/benchflow/providers/litellm_config.py`). Both keys are set on the\n  environment. If they are ever absent, credential-aware emission\n  ([`../scripts/filter_credentialed_cells.py`](../scripts/filter_credentialed_cells.py))\n  **drops** the `claude` cells as a documented skip (not a red slot).\n\n> **Roster & gating (current policy).** The broad fan is **DeepSeek-only**: the 5\n> DeepSeek agents (`openhands`, `pi-acp`, `openclaw`, `opencode`, `mimo`) run in\n> every roster lane. The 2 **gated natives** — `claude-agent-acp` (Bedrock) and\n> `codex-acp` (OpenAI) — cannot use DeepSeek and are blocked from the default\n> fan; they run **only via affected-agent** (a PR touching their own adapter), so\n> a default run needs only `DEEPSEEK_*` (rollout **and** judge) + `DAYTONA_API_KEY`.\n> `gemini` and `harvey-lab-harness` were removed from the roster.\n\nSandbox: `DAYTONA_API_KEY` (L2/L3 only; L1 is docker-only).\n\nCodex reviewer (L3, required): the **existing repo `OPENAI_API_KEY`** — durable\nand revocable, unlike a personal ChatGPT OAuth blob. `codex_review.py` writes it\nas an apikey `auth.json` at the codex config path (`$CODEX_HOME/auth.json`,\ndefault `~/.codex/auth.json`) before invoking `codex exec`. Auth precedence:\n`OPENAI_API_KEY` / `CODEX_API_KEY`, then an optional `CODEX_AUTH_JSON` blob, then\na pre-existing on-host `auth.json`. If none resolve, L3 fails closed to\n**`not mergeable (codex unavailable)`**.\n\n## 4. Labels\n\n- `integration:medium` — opens the L2 lane on a PR (its `detect-scope` only does\n  real work when this label is present).\n- `integration:deep` — signals an L2/L3-class change; L1's `rollout-smoke` steps\n  aside when present so the lanes don't double-spend provider credits.\n\n## 5. Security invariants (do not weaken)\n\n- PR-facing workflows use plain `pull_request`, never `pull_request_target`, so\n  forks receive no secrets. L1 also uses `workflow_run` after `test`, but its\n  environment-backed job is fail-closed to a successful `push` on `main` and\n  pins both trusted infrastructure and code-under-test to that tested SHA.\n- `detect-scope`, `plan`, and `review-pack` check out the planner / grader /\n  harness / review skill from **trusted `origin/main`** (sparse), never the PR\n  head. Only `src/benchflow` (the orchestrator code-under-test) is overlaid from\n  the PR head SHA.\n- The L3 final verdict is `worst(deterministic, codex)`: codex can only make the\n  verdict stricter, never upgrade a deterministic `not mergeable`.\n- Actions are SHA-pinned. `actions/download-artifact`\n  (`d3f86a106a0bac45b974a628896c90dbdf5c8093`, v4.3.0) and\n  `actions/github-script` (`60a0d83039c74a4aee543508d2ffcb1c3799cdea`, v7.0.1)\n  are new surface relative to L0/L1's existing pins — re-verify on bump.\n"}}, ".github/integration/scope_defaults.yml": {"cf656179f3bc30b2887b744cf71b69b5e5ffa5ff": {"t": 1785747016.6919615, "n": 8250, "h": "49fd95ae7aea2825", "c": "# Scope defaults — caps + baseline anchors for the deterministic matrix planner.\n#\n# Pure data consumed by .github/scripts/integration_matrix.py. No secrets, no\n# benchflow import. Mirrors the SPEC caps block, the AGGREGATE CONCURRENCY rule\n# (per_agent_concurrency * distinct daytona agents <= 24), and the baseline\n# anchors (baseline agent, canonical high task, citation vehicle, judge model).\n\nschema_version: \"1\"\n\n# ------------------------------------------------------------------\n# Caps. enforce_caps() rejects up front if caps are internally inconsistent,\n# and sets rejected_overflow (exit 2) if the enumerated matrix exceeds\n# max_cells. Self-consistent for `nine`: max_cells >= max_agents * max_tasks is\n# NOT required because a `nine` matrix carries all tasks in one cell's include\n# per agent (one cell per agent). But the SPEC requires the caps hold the\n# largest single-axis enumeration, so max_cells >= max_agents * max_tasks for\n# the worst-case per-(agent,task) expansion the planner may emit.\n# ------------------------------------------------------------------\ncaps:\n  # Sized to hold the largest legitimate lane: the `expanded` release-critical\n  # matrix = 10 tasks * 5 DeepSeek agents (broad fan) + per-task baseline\n  # axis-coverage cells across the extra (sandbox, skill_mode) combos + cheat,\n  # with headroom. The floor (max_agents * max_tasks = 63) is comfortably held.\n  # The ceiling still fail-closes a runaway custom override (rejected_overflow).\n  max_cells: 140\n  max_agents: 7\n  max_tasks: 9\n  # Per-agent rollout concurrency. Lowered automatically by the planner as the\n  # distinct daytona-agent count rises so aggregate stays under the ceiling.\n  per_agent_concurrency: 2\n  # Aggregate ceiling: per_agent_concurrency * distinct daytona agents <= 24.\n  aggregate_concurrency: 24\n  agent_idle_timeout: 240\n  # Trials per cell on the heavy (comment-triggered) tier.\n  comment_trials_cap: 3\n\n# ------------------------------------------------------------------\n# Baseline anchors.\n# ------------------------------------------------------------------\n# The agents/ rule pairs each affected agent with this baseline agent + model\n# so before/after comparison always has a stable reference point.\nbaseline_agent: openhands\nbaseline_model: deepseek/deepseek-v4-flash\n\n# The canonical \"one high task\" added to the agents/ rule's low-3 set.\ncanonical_high_task: weighted-gdp-calc\n\n# The citation vehicle task (citation lane + network allowlist variant).\ncitation_vehicle: citation-check\n\n# Default judge model. The agent-judge and the review-pack per-rollout reader run\n# on DeepSeek-v4 ONLY — no other models. At runtime select_integration_provider.py\n# exports BENCHFLOW_JUDGE_MODEL=openai/deepseek-v4-flash (routed via the OpenAI-\n# compatible chat-completions surface with the DeepSeek key + base_url); this\n# value is the recorded per-cell metadata and the local-run default. The judge is\n# NOT a matrix cell, so the credential filter does not gate it — DEEPSEEK_API_KEY\n# + DEEPSEEK_BASE_URL must be present (they are, as required CI secrets).\njudge_model: deepseek-v4-flash\n\n# Default agent model when a cell's agent has no explicit per-agent model. Every\n# roster agent has an explicit entry below, so this is only a safety net; it\n# points at the cheap DeepSeek flash lane.\ndefault_model: deepseek/deepseek-v4-flash\n\n# Per-agent default models (mirror release axes + run.sh model_for_agent).\n#\n# Empirically-grounded model policy — DeepSeek-first, native only when forced:\n#  - DeepSeek lane (openhands, openclaw, opencode, pi-acp, mimo) runs on\n#    deepseek/deepseek-v4-flash (needs DEEPSEEK_API_KEY + DEEPSEEK_BASE_URL, both\n#    in CI), promoted to deepseek-v4-pro on hard tasks via deepseek_tiering.\n#    mimo-on-deepseek REPLACES the old xiaomi/mimo id (closes the XIAOMI gap).\n#  - codex-acp keeps native gpt (OpenAI Responses) and claude-agent-acp routes\n#    through Bedrock's anthropic-messages surface. Both CANNOT use DeepSeek\n#    (openai-responses / anthropic-messages protocol guards), so they are GATED:\n#    fanned ONLY via affected-agent (their own adapter changed), never in the\n#    broad DeepSeek lanes. See deepseek_roster below.\n#  - gemini (native Google GenerateContent) and harvey-lab-harness (OpenAI\n#    Responses API) are DROPPED from the roster: neither can run on DeepSeek.\nagent_models:\n  claude-agent-acp: aws-bedrock/us.anthropic.claude-haiku-4-5-20251001\n  pi-acp: deepseek/deepseek-v4-flash\n  openclaw: deepseek/deepseek-v4-flash\n  codex-acp: gpt-5.4-nano\n  opencode: deepseek/deepseek-v4-flash\n  openhands: deepseek/deepseek-v4-flash\n  mimo: deepseek/deepseek-v4-flash\n\n# DeepSeek roster — the agents fanned in the BROAD lanes (all-agents at\n# L3/expanded, and the all-agents-subset breadth tier at L2). The gated native\n# agents (claude-agent-acp, codex-acp) are deliberately ABSENT here: they run\n# only via affected-agent, \"blocking\" non-DeepSeek agents from the default\n# real-workflow fan. To un-gate one later, add it back to this list.\ndeepseek_roster:\n  - openhands\n  - pi-acp\n  - openclaw\n  - opencode\n  - mimo\n\n# Breadth-tiered roster SUBSET emitted at L2 (auto-on-push) for changes that\n# affect EVERY agent. A representative DeepSeek subset (one per launcher family:\n# openhands = baseline/OpenHands, pi-acp = ACP launcher, opencode = opencode\n# proxy family). The FULL DeepSeek roster runs at L3/expanded.\nroster_subset:\n  - openhands\n  - pi-acp\n  - opencode\n\n# ------------------------------------------------------------------\n# DeepSeek model tiering by task difficulty.\n# The DeepSeek-driven baseline rollout uses the cheap `flash` model for\n# low/medium tasks and the stronger `pro` model for high-difficulty tasks\n# (the high-3 set). Tiering applies ONLY to cells whose resolved model is the\n# flash model (i.e. the openhands/baseline DeepSeek lane); every other agent\n# keeps its own per-agent model. If `pro_tasks` is omitted the planner falls\n# back to task_sets[\"high-3\"].\n# ------------------------------------------------------------------\ndeepseek_tiering:\n  flash_model: deepseek/deepseek-v4-flash\n  pro_model: deepseek/deepseek-v4-pro\n  pro_tasks:\n    - lake-warming-attribution\n    - weighted-gdp-calc\n    - shock-analysis-supply\n\n# ------------------------------------------------------------------\n# Per-set timeout_minutes for each cell (wall-clock cap on one rollout cell).\n# citation/low lanes are cheap; medium mid; high/nine heavy; expanded heaviest.\n# ------------------------------------------------------------------\ntimeout_minutes:\n  citation: 45\n  low-smoke: 45\n  low-3: 45\n  medium-3: 90\n  high-3: 180\n  nine: 180\n  expanded: 330\n  custom: 90\n\n# ------------------------------------------------------------------\n# Before/after reward-band baseline (golden truth) — Q2b.\n# The SkillsBench leaderboard submissions are native BenchFlow result.json\n# (rewards.reward), referenced at the latest leaderboard `main`. The L3 review\n# job fetches the openhands deepseek-v4-flash run to a local root and passes it\n# to build_integration_review_pack.py --baseline-root, which band-compares\n# per-task reward via check_skillsbench_harbor_parity. Apples-to-apples with our\n# openhands + deepseek-v4-flash lane. No protected env is created — the golden\n# truth is simply this HF `main` run plus the latest benchflow main.\n#\n# Coverage: the flash run covers 6 of the 9 integration tasks (data-to-d3,\n# grid-dispatch-operator, jax-computing-basics, jpg-ocr-stat,\n# lake-warming-attribution, threejs-to-obj). The other 3 (python-scala-\n# translation, shock-analysis-supply, weighted-gdp-calc) are absent from the\n# leaderboard's deepseek runs, so they band-compare NA and fall back to oracle\n# determinism + the codex equivalence verdict. High tasks run the pro model\n# (tiering), so their flash-baseline comparison is advisory.\n# ------------------------------------------------------------------\nbaseline:\n  source: huggingface\n  hf_repo: benchflow/skillsbench-leaderboard\n  hf_revision: main\n  submission_root: submissions/skillsbench/v1.1\n  config_no_skill: openhands-no-skills\n  config_with_skill: openhands-with-skills\n  band_model: deepseek-v4-flash\n  max_task_reward_delta: 0.0\n  max_mean_reward_delta: 0.25\n  max_outcome_rate_delta: 0.25\n"}, "e1060f6cc3eca6226f60971be9b2dc340763c15a": {"t": 1785749195.3982236, "n": 8250, "h": "49fd95ae7aea2825", "c": "# Scope defaults — caps + baseline anchors for the deterministic matrix planner.\n#\n# Pure data consumed by .github/scripts/integration_matrix.py. No secrets, no\n# benchflow import. Mirrors the SPEC caps block, the AGGREGATE CONCURRENCY rule\n# (per_agent_concurrency * distinct daytona agents <= 24), and the baseline\n# anchors (baseline agent, canonical high task, citation vehicle, judge model).\n\nschema_version: \"1\"\n\n# ------------------------------------------------------------------\n# Caps. enforce_caps() rejects up front if caps are internally inconsistent,\n# and sets rejected_overflow (exit 2) if the enumerated matrix exceeds\n# max_cells. Self-consistent for `nine`: max_cells >= max_agents * max_tasks is\n# NOT required because a `nine` matrix carries all tasks in one cell's include\n# per agent (one cell per agent). But the SPEC requires the caps hold the\n# largest single-axis enumeration, so max_cells >= max_agents * max_tasks for\n# the worst-case per-(agent,task) expansion the planner may emit.\n# ------------------------------------------------------------------\ncaps:\n  # Sized to hold the largest legitimate lane: the `expanded` release-critical\n  # matrix = 10 tasks * 5 DeepSeek agents (broad fan) + per-task baseline\n  # axis-coverage cells across the extra (sandbox, skill_mode) combos + cheat,\n  # with headroom. The floor (max_agents * max_tasks = 63) is comfortably held.\n  # The ceiling still fail-closes a runaway custom override (rejected_overflow).\n  max_cells: 140\n  max_agents: 7\n  max_tasks: 9\n  # Per-agent rollout concurrency. Lowered automatically by the planner as the\n  # distinct daytona-agent count rises so aggregate stays under the ceiling.\n  per_agent_concurrency: 2\n  # Aggregate ceiling: per_agent_concurrency * distinct daytona agents <= 24.\n  aggregate_concurrency: 24\n  agent_idle_timeout: 240\n  # Trials per cell on the heavy (comment-triggered) tier.\n  comment_trials_cap: 3\n\n# ------------------------------------------------------------------\n# Baseline anchors.\n# ------------------------------------------------------------------\n# The agents/ rule pairs each affected agent with this baseline agent + model\n# so before/after comparison always has a stable reference point.\nbaseline_agent: openhands\nbaseline_model: deepseek/deepseek-v4-flash\n\n# The canonical \"one high task\" added to the agents/ rule's low-3 set.\ncanonical_high_task: weighted-gdp-calc\n\n# The citation vehicle task (citation lane + network allowlist variant).\ncitation_vehicle: citation-check\n\n# Default judge model. The agent-judge and the review-pack per-rollout reader run\n# on DeepSeek-v4 ONLY — no other models. At runtime select_integration_provider.py\n# exports BENCHFLOW_JUDGE_MODEL=openai/deepseek-v4-flash (routed via the OpenAI-\n# compatible chat-completions surface with the DeepSeek key + base_url); this\n# value is the recorded per-cell metadata and the local-run default. The judge is\n# NOT a matrix cell, so the credential filter does not gate it — DEEPSEEK_API_KEY\n# + DEEPSEEK_BASE_URL must be present (they are, as required CI secrets).\njudge_model: deepseek-v4-flash\n\n# Default agent model when a cell's agent has no explicit per-agent model. Every\n# roster agent has an explicit entry below, so this is only a safety net; it\n# points at the cheap DeepSeek flash lane.\ndefault_model: deepseek/deepseek-v4-flash\n\n# Per-agent default models (mirror release axes + run.sh model_for_agent).\n#\n# Empirically-grounded model policy — DeepSeek-first, native only when forced:\n#  - DeepSeek lane (openhands, openclaw, opencode, pi-acp, mimo) runs on\n#    deepseek/deepseek-v4-flash (needs DEEPSEEK_API_KEY + DEEPSEEK_BASE_URL, both\n#    in CI), promoted to deepseek-v4-pro on hard tasks via deepseek_tiering.\n#    mimo-on-deepseek REPLACES the old xiaomi/mimo id (closes the XIAOMI gap).\n#  - codex-acp keeps native gpt (OpenAI Responses) and claude-agent-acp routes\n#    through Bedrock's anthropic-messages surface. Both CANNOT use DeepSeek\n#    (openai-responses / anthropic-messages protocol guards), so they are GATED:\n#    fanned ONLY via affected-agent (their own adapter changed), never in the\n#    broad DeepSeek lanes. See deepseek_roster below.\n#  - gemini (native Google GenerateContent) and harvey-lab-harness (OpenAI\n#    Responses API) are DROPPED from the roster: neither can run on DeepSeek.\nagent_models:\n  claude-agent-acp: aws-bedrock/us.anthropic.claude-haiku-4-5-20251001\n  pi-acp: deepseek/deepseek-v4-flash\n  openclaw: deepseek/deepseek-v4-flash\n  codex-acp: gpt-5.4-nano\n  opencode: deepseek/deepseek-v4-flash\n  openhands: deepseek/deepseek-v4-flash\n  mimo: deepseek/deepseek-v4-flash\n\n# DeepSeek roster — the agents fanned in the BROAD lanes (all-agents at\n# L3/expanded, and the all-agents-subset breadth tier at L2). The gated native\n# agents (claude-agent-acp, codex-acp) are deliberately ABSENT here: they run\n# only via affected-agent, \"blocking\" non-DeepSeek agents from the default\n# real-workflow fan. To un-gate one later, add it back to this list.\ndeepseek_roster:\n  - openhands\n  - pi-acp\n  - openclaw\n  - opencode\n  - mimo\n\n# Breadth-tiered roster SUBSET emitted at L2 (auto-on-push) for changes that\n# affect EVERY agent. A representative DeepSeek subset (one per launcher family:\n# openhands = baseline/OpenHands, pi-acp = ACP launcher, opencode = opencode\n# proxy family). The FULL DeepSeek roster runs at L3/expanded.\nroster_subset:\n  - openhands\n  - pi-acp\n  - opencode\n\n# ------------------------------------------------------------------\n# DeepSeek model tiering by task difficulty.\n# The DeepSeek-driven baseline rollout uses the cheap `flash` model for\n# low/medium tasks and the stronger `pro` model for high-difficulty tasks\n# (the high-3 set). Tiering applies ONLY to cells whose resolved model is the\n# flash model (i.e. the openhands/baseline DeepSeek lane); every other agent\n# keeps its own per-agent model. If `pro_tasks` is omitted the planner falls\n# back to task_sets[\"high-3\"].\n# ------------------------------------------------------------------\ndeepseek_tiering:\n  flash_model: deepseek/deepseek-v4-flash\n  pro_model: deepseek/deepseek-v4-pro\n  pro_tasks:\n    - lake-warming-attribution\n    - weighted-gdp-calc\n    - shock-analysis-supply\n\n# ------------------------------------------------------------------\n# Per-set timeout_minutes for each cell (wall-clock cap on one rollout cell).\n# citation/low lanes are cheap; medium mid; high/nine heavy; expanded heaviest.\n# ------------------------------------------------------------------\ntimeout_minutes:\n  citation: 45\n  low-smoke: 45\n  low-3: 45\n  medium-3: 90\n  high-3: 180\n  nine: 180\n  expanded: 330\n  custom: 90\n\n# ------------------------------------------------------------------\n# Before/after reward-band baseline (golden truth) — Q2b.\n# The SkillsBench leaderboard submissions are native BenchFlow result.json\n# (rewards.reward), referenced at the latest leaderboard `main`. The L3 review\n# job fetches the openhands deepseek-v4-flash run to a local root and passes it\n# to build_integration_review_pack.py --baseline-root, which band-compares\n# per-task reward via check_skillsbench_harbor_parity. Apples-to-apples with our\n# openhands + deepseek-v4-flash lane. No protected env is created — the golden\n# truth is simply this HF `main` run plus the latest benchflow main.\n#\n# Coverage: the flash run covers 6 of the 9 integration tasks (data-to-d3,\n# grid-dispatch-operator, jax-computing-basics, jpg-ocr-stat,\n# lake-warming-attribution, threejs-to-obj). The other 3 (python-scala-\n# translation, shock-analysis-supply, weighted-gdp-calc) are absent from the\n# leaderboard's deepseek runs, so they band-compare NA and fall back to oracle\n# determinism + the codex equivalence verdict. High tasks run the pro model\n# (tiering), so their flash-baseline comparison is advisory.\n# ------------------------------------------------------------------\nbaseline:\n  source: huggingface\n  hf_repo: benchflow/skillsbench-leaderboard\n  hf_revision: main\n  submission_root: submissions/skillsbench/v1.1\n  config_no_skill: openhands-no-skills\n  config_with_skill: openhands-with-skills\n  band_model: deepseek-v4-flash\n  max_task_reward_delta: 0.0\n  max_mean_reward_delta: 0.25\n  max_outcome_rate_delta: 0.25\n"}, "2631b1d47436fbaa6017956c2957e7d4147b3cae": {"t": 1786068535.5831132, "n": 8250, "h": "49fd95ae7aea2825", "c": "# Scope defaults — caps + baseline anchors for the deterministic matrix planner.\n#\n# Pure data consumed by .github/scripts/integration_matrix.py. No secrets, no\n# benchflow import. Mirrors the SPEC caps block, the AGGREGATE CONCURRENCY rule\n# (per_agent_concurrency * distinct daytona agents <= 24), and the baseline\n# anchors (baseline agent, canonical high task, citation vehicle, judge model).\n\nschema_version: \"1\"\n\n# ------------------------------------------------------------------\n# Caps. enforce_caps() rejects up front if caps are internally inconsistent,\n# and sets rejected_overflow (exit 2) if the enumerated matrix exceeds\n# max_cells. Self-consistent for `nine`: max_cells >= max_agents * max_tasks is\n# NOT required because a `nine` matrix carries all tasks in one cell's include\n# per agent (one cell per agent). But the SPEC requires the caps hold the\n# largest single-axis enumeration, so max_cells >= max_agents * max_tasks for\n# the worst-case per-(agent,task) expansion the planner may emit.\n# ------------------------------------------------------------------\ncaps:\n  # Sized to hold the largest legitimate lane: the `expanded` release-critical\n  # matrix = 10 tasks * 5 DeepSeek agents (broad fan) + per-task baseline\n  # axis-coverage cells across the extra (sandbox, skill_mode) combos + cheat,\n  # with headroom. The floor (max_agents * max_tasks = 63) is comfortably held.\n  # The ceiling still fail-closes a runaway custom override (rejected_overflow).\n  max_cells: 140\n  max_agents: 7\n  max_tasks: 9\n  # Per-agent rollout concurrency. Lowered automatically by the planner as the\n  # distinct daytona-agent count rises so aggregate stays under the ceiling.\n  per_agent_concurrency: 2\n  # Aggregate ceiling: per_agent_concurrency * distinct daytona agents <= 24.\n  aggregate_concurrency: 24\n  agent_idle_timeout: 240\n  # Trials per cell on the heavy (comment-triggered) tier.\n  comment_trials_cap: 3\n\n# ------------------------------------------------------------------\n# Baseline anchors.\n# ------------------------------------------------------------------\n# The agents/ rule pairs each affected agent with this baseline agent + model\n# so before/after comparison always has a stable reference point.\nbaseline_agent: openhands\nbaseline_model: deepseek/deepseek-v4-flash\n\n# The canonical \"one high task\" added to the agents/ rule's low-3 set.\ncanonical_high_task: weighted-gdp-calc\n\n# The citation vehicle task (citation lane + network allowlist variant).\ncitation_vehicle: citation-check\n\n# Default judge model. The agent-judge and the review-pack per-rollout reader run\n# on DeepSeek-v4 ONLY — no other models. At runtime select_integration_provider.py\n# exports BENCHFLOW_JUDGE_MODEL=openai/deepseek-v4-flash (routed via the OpenAI-\n# compatible chat-completions surface with the DeepSeek key + base_url); this\n# value is the recorded per-cell metadata and the local-run default. The judge is\n# NOT a matrix cell, so the credential filter does not gate it — DEEPSEEK_API_KEY\n# + DEEPSEEK_BASE_URL must be present (they are, as required CI secrets).\njudge_model: deepseek-v4-flash\n\n# Default agent model when a cell's agent has no explicit per-agent model. Every\n# roster agent has an explicit entry below, so this is only a safety net; it\n# points at the cheap DeepSeek flash lane.\ndefault_model: deepseek/deepseek-v4-flash\n\n# Per-agent default models (mirror release axes + run.sh model_for_agent).\n#\n# Empirically-grounded model policy — DeepSeek-first, native only when forced:\n#  - DeepSeek lane (openhands, openclaw, opencode, pi-acp, mimo) runs on\n#    deepseek/deepseek-v4-flash (needs DEEPSEEK_API_KEY + DEEPSEEK_BASE_URL, both\n#    in CI), promoted to deepseek-v4-pro on hard tasks via deepseek_tiering.\n#    mimo-on-deepseek REPLACES the old xiaomi/mimo id (closes the XIAOMI gap).\n#  - codex-acp keeps native gpt (OpenAI Responses) and claude-agent-acp routes\n#    through Bedrock's anthropic-messages surface. Both CANNOT use DeepSeek\n#    (openai-responses / anthropic-messages protocol guards), so they are GATED:\n#    fanned ONLY via affected-agent (their own adapter changed), never in the\n#    broad DeepSeek lanes. See deepseek_roster below.\n#  - gemini (native Google GenerateContent) and harvey-lab-harness (OpenAI\n#    Responses API) are DROPPED from the roster: neither can run on DeepSeek.\nagent_models:\n  claude-agent-acp: aws-bedrock/us.anthropic.claude-haiku-4-5-20251001\n  pi-acp: deepseek/deepseek-v4-flash\n  openclaw: deepseek/deepseek-v4-flash\n  codex-acp: gpt-5.4-nano\n  opencode: deepseek/deepseek-v4-flash\n  openhands: deepseek/deepseek-v4-flash\n  mimo: deepseek/deepseek-v4-flash\n\n# DeepSeek roster — the agents fanned in the BROAD lanes (all-agents at\n# L3/expanded, and the all-agents-subset breadth tier at L2). The gated native\n# agents (claude-agent-acp, codex-acp) are deliberately ABSENT here: they run\n# only via affected-agent, \"blocking\" non-DeepSeek agents from the default\n# real-workflow fan. To un-gate one later, add it back to this list.\ndeepseek_roster:\n  - openhands\n  - pi-acp\n  - openclaw\n  - opencode\n  - mimo\n\n# Breadth-tiered roster SUBSET emitted at L2 (auto-on-push) for changes that\n# affect EVERY agent. A representative DeepSeek subset (one per launcher family:\n# openhands = baseline/OpenHands, pi-acp = ACP launcher, opencode = opencode\n# proxy family). The FULL DeepSeek roster runs at L3/expanded.\nroster_subset:\n  - openhands\n  - pi-acp\n  - opencode\n\n# ------------------------------------------------------------------\n# DeepSeek model tiering by task difficulty.\n# The DeepSeek-driven baseline rollout uses the cheap `flash` model for\n# low/medium tasks and the stronger `pro` model for high-difficulty tasks\n# (the high-3 set). Tiering applies ONLY to cells whose resolved model is the\n# flash model (i.e. the openhands/baseline DeepSeek lane); every other agent\n# keeps its own per-agent model. If `pro_tasks` is omitted the planner falls\n# back to task_sets[\"high-3\"].\n# ------------------------------------------------------------------\ndeepseek_tiering:\n  flash_model: deepseek/deepseek-v4-flash\n  pro_model: deepseek/deepseek-v4-pro\n  pro_tasks:\n    - lake-warming-attribution\n    - weighted-gdp-calc\n    - shock-analysis-supply\n\n# ------------------------------------------------------------------\n# Per-set timeout_minutes for each cell (wall-clock cap on one rollout cell).\n# citation/low lanes are cheap; medium mid; high/nine heavy; expanded heaviest.\n# ------------------------------------------------------------------\ntimeout_minutes:\n  citation: 45\n  low-smoke: 45\n  low-3: 45\n  medium-3: 90\n  high-3: 180\n  nine: 180\n  expanded: 330\n  custom: 90\n\n# ------------------------------------------------------------------\n# Before/after reward-band baseline (golden truth) — Q2b.\n# The SkillsBench leaderboard submissions are native BenchFlow result.json\n# (rewards.reward), referenced at the latest leaderboard `main`. The L3 review\n# job fetches the openhands deepseek-v4-flash run to a local root and passes it\n# to build_integration_review_pack.py --baseline-root, which band-compares\n# per-task reward via check_skillsbench_harbor_parity. Apples-to-apples with our\n# openhands + deepseek-v4-flash lane. No protected env is created — the golden\n# truth is simply this HF `main` run plus the latest benchflow main.\n#\n# Coverage: the flash run covers 6 of the 9 integration tasks (data-to-d3,\n# grid-dispatch-operator, jax-computing-basics, jpg-ocr-stat,\n# lake-warming-attribution, threejs-to-obj). The other 3 (python-scala-\n# translation, shock-analysis-supply, weighted-gdp-calc) are absent from the\n# leaderboard's deepseek runs, so they band-compare NA and fall back to oracle\n# determinism + the codex equivalence verdict. High tasks run the pro model\n# (tiering), so their flash-baseline comparison is advisory.\n# ------------------------------------------------------------------\nbaseline:\n  source: huggingface\n  hf_repo: benchflow/skillsbench-leaderboard\n  hf_revision: main\n  submission_root: submissions/skillsbench/v1.1\n  config_no_skill: openhands-no-skills\n  config_with_skill: openhands-with-skills\n  band_model: deepseek-v4-flash\n  max_task_reward_delta: 0.0\n  max_mean_reward_delta: 0.25\n  max_outcome_rate_delta: 0.25\n"}}}}