{"repo": "casia-lm/s1-nexusagent", "files": {"deepagents/.github/workflows/evals.yml": {"be56be5e22b2ca1f4e28011bd81a8e55796b4b73": {"t": 1786067330.5770102, "n": 6279, "h": "cc32b2f0b341c01c", "c": "# Daily evaluation workflow for Deep Agents\n#\n# Runs tests/evals on a cron schedule (once per day).\n# Single job; model/provider is selected via workflow input `model`.\n#\n# Required secrets:\n#   LANGSMITH_API_KEY  — used for tracing\n#   ANTHROPIC_API_KEY  — needed for Anthropic models\n#   OPENAI_API_KEY     — needed for OpenAI models\n#   GOOGLE_API_KEY     — needed for Google models\n#   XAI_API_KEY        — needed for xAI/Grok models\n#   MISTRAL_API_KEY    — needed for Mistral models\n#   DEEPSEEK_API_KEY   — needed for DeepSeek models\n#   GROQ_API_KEY       — needed for Groq-hosted models\n#   OLLAMA_API_KEY     — needed for Ollama Cloud models\n#   OLLAMA_HOST        — set to https://ollama.com for cloud inference\n#   BASETEN_API_KEY    — needed for Baseten-hosted models\n#   FIREWORKS_API_KEY  — needed for Fireworks-hosted models\n\nname: \"📊 Evals\"\n\non:\n  schedule:\n    - cron: \"0 0 * * *\"\n  workflow_dispatch:\n    inputs:\n      models:\n        description: \"Preset model selection\"\n        required: true\n        default: \"all\"\n        type: choice\n        options:\n          - all\n          - set0\n          - set1\n          - set2\n          - \"anthropic:claude-haiku-4-5-20251001\"\n          - \"anthropic:claude-sonnet-4-20250514\"\n          - \"anthropic:claude-sonnet-4-5-20250929\"\n          - \"anthropic:claude-sonnet-4-6\"\n          - \"anthropic:claude-opus-4-1\"\n          - \"anthropic:claude-opus-4-5-20251101\"\n          - \"anthropic:claude-opus-4-6\"\n          - \"openai:gpt-4o\"\n          - \"openai:gpt-4o-mini\"\n          - \"openai:gpt-4.1\"\n          - \"openai:o3\"\n          - \"openai:o4-mini\"\n          - \"openai:gpt-5.1-codex\"\n          - \"openai:gpt-5.2-codex\"\n          - \"openai:gpt-5.4\"\n          - \"google_genai:gemini-2.5-flash\"\n          - \"google_genai:gemini-2.5-pro\"\n          - \"google_genai:gemini-3-flash-preview\"\n          - \"google_genai:gemini-3.1-pro-preview\"\n          - \"xai:grok-4\"\n          - \"xai:grok-3-mini-fast\"\n          - \"groq:openai/gpt-oss-120b\"\n          - \"groq:qwen/qwen3-32b\"\n          - \"groq:moonshotai/kimi-k2-instruct\"\n          - \"ollama:glm-5\"\n          - \"ollama:minimax-m2.5\"\n          - \"ollama:nemotron-3-nano:30b\"\n          - \"ollama:cogito-2.1:671b\"\n          - \"ollama:devstral-2:123b\"\n          - \"ollama:ministral-3:14b\"\n          - \"ollama:qwen3-next:80b\"\n          - \"ollama:qwen3-coder:480b-cloud\"\n          - \"ollama:qwen3.5:397b-cloud\"\n          - \"ollama:deepseek-v3.2:cloud\"\n          - \"baseten:zai-org/GLM-5\"\n          - \"baseten:MiniMaxAI/MiniMax-M2.5\"\n          - \"baseten:moonshotai/Kimi-K2.5\"\n          - \"baseten:deepseek-ai/DeepSeek-V3.2\"\n          - \"baseten:Qwen/Qwen3-Coder-480B-A35B-Instruct\"\n          - \"fireworks:fireworks/qwen3-vl-235b-a22b-thinking\"\n          - \"fireworks:fireworks/deepseek-v3-0324\"\n          - \"fireworks:fireworks/minimax-m2p1\"\n          - \"fireworks:fireworks/kimi-k2p5\"\n          - \"fireworks:fireworks/glm-5\"\n          - \"fireworks:fireworks/minimax-m2p5\"\n      models_override:\n        description: \"Override: comma-separated models (e.g. 'openai:gpt-4.1,openai:o3'). Takes priority over dropdown when non-empty.\"\n        required: false\n        default: \"\"\n        type: string\n\npermissions:\n  contents: read\n\nenv:\n  UV_NO_SYNC: \"true\"\n\njobs:\n  prep:\n    name: \"🔧 Prepare matrix\"\n    runs-on: ubuntu-latest\n    outputs:\n      matrix: ${{ steps.set-matrix.outputs.matrix }}\n    steps:\n      - name: \"📋 Checkout Code\"\n        uses: actions/checkout@v6\n\n      - name: \"🐍 Compute eval matrix\"\n        id: set-matrix\n        run: python .github/scripts/get_eval_models.py\n        env:\n          EVAL_MODELS: ${{ inputs.models_override || inputs.models || 'all' }}\n\n  eval:\n    name: \"📊 Eval (${{ matrix.model }})\"\n    needs: prep\n    runs-on: ubuntu-latest\n    timeout-minutes: 120\n    strategy:\n      fail-fast: false\n      matrix: ${{ fromJson(needs.prep.outputs.matrix) }}\n\n    defaults:\n      run:\n        working-directory: libs/deepagents\n    env:\n      PYTEST_ADDOPTS: \"--model ${{ matrix.model }} --evals-report-file evals_report.json\"\n      LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}\n      LANGSMITH_TRACING_V2: \"true\"\n      LANGSMITH_EXPERIMENT: ${{ matrix.model }}\n      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}\n      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}\n      XAI_API_KEY: ${{ secrets.XAI_API_KEY }}\n      MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}\n      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n      GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}\n      OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}\n      OLLAMA_HOST: \"https://ollama.com\"\n      BASETEN_API_KEY: ${{ secrets.BASETEN_API_KEY }}\n      FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}\n    steps:\n      - name: \"📋 Checkout Code\"\n        uses: actions/checkout@v6\n\n      - name: \"🐍 Set up Python + UV\"\n        uses: \"./.github/actions/uv_setup\"\n        with:\n          python-version: \"3.11\"\n          cache-suffix: evals\n          working-directory: libs/deepagents\n\n      - name: \"📦 Install Dependencies\"\n        run: uv sync --group test --group evals --locked\n\n      - name: \"📊 Run Evals\"\n        run: make evals\n\n      - name: \"📤 Upload eval report\"\n        if: always()\n        uses: actions/upload-artifact@v7\n        with:\n          name: evals-report-${{ strategy.job-index }}\n          path: libs/deepagents/evals_report.json\n          if-no-files-found: error\n\n  aggregate:\n    name: \"📋 Aggregate evals\"\n    runs-on: ubuntu-latest\n    needs: [eval]\n    if: always()\n    steps:\n      - name: \"📋 Checkout Code\"\n        uses: actions/checkout@v6\n\n      - name: \"📥 Download eval artifacts\"\n        uses: actions/download-artifact@v7\n        with:\n          path: evals_artifacts\n\n      - name: \"🐍 Set up Python + UV\"\n        uses: \"./.github/actions/uv_setup\"\n        with:\n          python-version: \"3.11\"\n          cache-suffix: evals-aggregate\n          working-directory: libs/deepagents\n\n      - name: \"🧾 Write summary\"\n        run: uv run --with tabulate python .github/scripts/aggregate_evals.py\n\n      - name: \"📤 Upload JSON summary\"\n        uses: actions/upload-artifact@v7\n        with:\n          name: evals-summary\n          path: evals_summary.json\n          if-no-files-found: warn\n"}}}}