1 line
15 KiB
JSON
1 line
15 KiB
JSON
{"repo": "avishkar1117/rag-as-a-service", "files": {".env.example": {"0ee26332584f20b38811321e70af159e445684a8": {"t": 1785745148.0997486, "n": 1732, "h": "0cc429d043236af9", "c": "GEMINI_API_KEY=your_gemini_api_key_here\n\n# Answer-generation model (Gemini API, free-tier Gemma).\nGEMMA_MODEL=gemma-4-31b-it\n\n# --- RAGAS judge ------------------------------------------------------------\n# JUDGE_PROVIDER selects which provider's credentials are used for the RAGAS\n# judge. Valid values: \"deepseek\" | \"openrouter\" | \"gemini\". Default is\n# DeepSeek (separate provider + model family from the Gemma generator).\nJUDGE_PROVIDER=deepseek\n\n# DeepSeek platform — fill this in for JUDGE_PROVIDER=deepseek.\n# Available models: deepseek-v4-flash (default), deepseek-v4-pro,\n# or legacy deepseek-chat / deepseek-reasoner (deprecated 2026-07-24).\nDEEPSEEK_API_KEY=your_deepseek_api_key_here\n# DEEPSEEK_BASE_URL=https://api.deepseek.com\n# DEEPSEEK_JUDGE_MODEL=deepseek-v4-flash\n\n# OpenRouter — fill OPENROUTER_API_KEY to enable JUDGE_PROVIDER=openrouter.\n# OPENROUTER_API_KEY=your_openrouter_api_key_here\n# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1\n# OPENROUTER_JUDGE_MODEL=nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free\n\n# Gemini-as-judge — same family as the generator, reintroduces self-bias.\n# Uses GEMINI_API_KEY above. Only override these to point at a different\n# Gemini model.\n# GEMINI_JUDGE_BASE_URL=https://generativelanguage.googleapis.com/v1beta/openai/\n# GEMINI_JUDGE_MODEL=gemma-4-31b-it\n\n# OCR model (scanned-page text extraction) and the text-embedding model.\nOCR_MODEL=gemini-3.1-flash-lite\nEMBEDDING_MODEL=models/gemini-embedding-001\n\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=8\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}, "b034ac92bfe2c6a9fbb8a5ab270bbcc7d893371c": {"t": 1785748182.0814633, "n": 1732, "h": "0cc429d043236af9", "c": "GEMINI_API_KEY=your_gemini_api_key_here\n\n# Answer-generation model (Gemini API, free-tier Gemma).\nGEMMA_MODEL=gemma-4-31b-it\n\n# --- RAGAS judge ------------------------------------------------------------\n# JUDGE_PROVIDER selects which provider's credentials are used for the RAGAS\n# judge. Valid values: \"deepseek\" | \"openrouter\" | \"gemini\". Default is\n# DeepSeek (separate provider + model family from the Gemma generator).\nJUDGE_PROVIDER=deepseek\n\n# DeepSeek platform — fill this in for JUDGE_PROVIDER=deepseek.\n# Available models: deepseek-v4-flash (default), deepseek-v4-pro,\n# or legacy deepseek-chat / deepseek-reasoner (deprecated 2026-07-24).\nDEEPSEEK_API_KEY=your_deepseek_api_key_here\n# DEEPSEEK_BASE_URL=https://api.deepseek.com\n# DEEPSEEK_JUDGE_MODEL=deepseek-v4-flash\n\n# OpenRouter — fill OPENROUTER_API_KEY to enable JUDGE_PROVIDER=openrouter.\n# OPENROUTER_API_KEY=your_openrouter_api_key_here\n# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1\n# OPENROUTER_JUDGE_MODEL=nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free\n\n# Gemini-as-judge — same family as the generator, reintroduces self-bias.\n# Uses GEMINI_API_KEY above. Only override these to point at a different\n# Gemini model.\n# GEMINI_JUDGE_BASE_URL=https://generativelanguage.googleapis.com/v1beta/openai/\n# GEMINI_JUDGE_MODEL=gemma-4-31b-it\n\n# OCR model (scanned-page text extraction) and the text-embedding model.\nOCR_MODEL=gemini-3.1-flash-lite\nEMBEDDING_MODEL=models/gemini-embedding-001\n\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=8\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}, "d0fb277fb2cdcddb5b1533fe465849ce2e1e01bc": {"t": 1785748183.363307, "n": 842, "h": "7ed4fb06bbdc28f5", "c": "GEMINI_API_KEY=your_gemini_api_key_here\n\n# Answer-generation model (Gemini API, free-tier Gemma).\nGEMMA_MODEL=gemma-4-31b-it\n\n# RAGAS judge runs on OpenRouter — separate provider + model family from\n# the generator, to reduce self-bias on faithfulness scoring.\nRAGAS_JUDGE_MODEL=nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free\nOPENROUTER_API_KEY=your_openrouter_api_key_here\nOPENROUTER_BASE_URL=https://openrouter.ai/api/v1\n\n# OCR model (scanned-page text extraction) and the text-embedding model.\nOCR_MODEL=gemini-3.1-flash-lite\nEMBEDDING_MODEL=models/gemini-embedding-001\n\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=8\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}, "7098386b8600ed5d3b833193ee470a0faea10412": {"t": 1785748184.4043372, "n": 677, "h": "acad00999453590a", "c": "GEMINI_API_KEY=your_gemini_api_key_here\n\n# Answer-generation model, and the model RAGAS uses to judge eval metrics.\n# Both default to Gemma for its generous free-tier quota; both have working defaults.\nGEMMA_MODEL=gemma-4-31b-it\nRAGAS_JUDGE_MODEL=gemma-4-31b-it\n\n# OCR model (scanned-page text extraction) and the text-embedding model.\nOCR_MODEL=gemini-3.1-flash-lite\nEMBEDDING_MODEL=models/gemini-embedding-001\n\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=8\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}, "9bb1b62706144477cdeae57fc7aa06bf3fb5a70e": {"t": 1785748185.4861908, "n": 675, "h": "3b982d8c49c2476d", "c": "GEMINI_API_KEY=your_gemini_api_key_here\n\n# Answer-generation model, and the model RAGAS uses to judge eval metrics.\n# Both default to Gemma for its generous free-tier quota; both have working defaults.\nGEMMA_MODEL=gemma-4-31b-it\nRAGAS_JUDGE_MODEL=gemma-4-31b-it\n\n# OCR model (scanned-page text extraction) and the text-embedding model.\nOCR_MODEL=gemini-3.1-flash-lite\nEMBEDDING_MODEL=models/gemini-embedding-2\n\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=8\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}, "ed27a81a1b77fd6481cffdce3e1ff5e1bdd77a27": {"t": 1785748186.7612212, "n": 527, "h": "dcb5ade6c78ff5bc", "c": "GEMINI_API_KEY=your_gemini_api_key_here\n\n# Answer-generation model, and the model RAGAS uses to judge eval metrics.\n# Both default to Gemma for its generous free-tier quota; both have working defaults.\nGEMMA_MODEL=gemma-4-31b-it\nRAGAS_JUDGE_MODEL=gemma-4-31b-it\n\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=8\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}, "2df7038ca6d1f001c7c6c93b099b47e2a68287fb": {"t": 1785748187.8583848, "n": 527, "h": "9b878dbbc786972c", "c": "GEMINI_API_KEY=your_gemini_api_key_here\n\n# Answer-generation model, and the model RAGAS uses to judge eval metrics.\n# Both default to Gemma for its generous free-tier quota; both have working defaults.\nGEMMA_MODEL=gemma-4-31b-it\nRAGAS_JUDGE_MODEL=gemma-4-31b-it\n\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=3\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}, "87b772a4d537cc97a6448e2688419d08b88e4b59": {"t": 1785748188.9712276, "n": 304, "h": "92d877119b690228", "c": "GEMINI_API_KEY=your_gemini_api_key_here\nGITHUB_TOKEN=your_github_token_here\nGITHUB_MODEL=gpt-4o\nGITHUB_API_BASE=https://models.inference.ai.azure.com\n\nREDIS_URL=redis://localhost:6379\nCHROMA_PERSIST_DIR=./chroma_store\n\nSENTRY_DSN=\n\nCHUNK_SIZE=1024\nCHUNK_OVERLAP=50\nTOP_K=3\nCACHE_TTL=86400\nLOG_LEVEL=INFO\n"}}, ".github/workflows/eval.yml": {"0ee26332584f20b38811321e70af159e445684a8": {"t": 1785747036.4201157, "n": 1483, "h": "772375005978e5d0", "c": "name: Eval Regression\n\n# Evaluation as CI: every PR runs RAGAS over a 10-question subset and the\n# job fails if faithfulness regresses. Also runnable on demand.\non:\n pull_request:\n workflow_dispatch:\n\n# A new commit on the same branch supersedes an in-flight eval run.\nconcurrency:\n group: eval-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n ragas-regression:\n runs-on: ubuntu-latest\n steps:\n # lfs: true pulls the eval corpus PDFs — they are stored via Git LFS,\n # so a default checkout would yield pointer files and the eval would fail.\n - uses: actions/checkout@v4\n with:\n lfs: true\n\n - name: Install uv\n uses: astral-sh/setup-uv@v5\n with:\n version: \"0.5.0\"\n enable-cache: true\n\n - name: Install dependencies\n run: uv sync --frozen --python 3.10\n\n # Needs GEMINI_API_KEY (generation + embedding) and the API key for\n # whichever JUDGE_PROVIDER is active. All judge keys are forwarded\n # unconditionally so a future provider swap is just a JUDGE_PROVIDER\n # change in the workflow env / repo variables — no .yml edit needed.\n # Missing keys cause the regression test to skip rather than fail.\n - name: RAGAS regression eval\n env:\n GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}\n run: uv run pytest -m eval -v\n"}, "b034ac92bfe2c6a9fbb8a5ab270bbcc7d893371c": {"t": 1785748181.684146, "n": 1483, "h": "772375005978e5d0", "c": "name: Eval Regression\n\n# Evaluation as CI: every PR runs RAGAS over a 10-question subset and the\n# job fails if faithfulness regresses. Also runnable on demand.\non:\n pull_request:\n workflow_dispatch:\n\n# A new commit on the same branch supersedes an in-flight eval run.\nconcurrency:\n group: eval-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n ragas-regression:\n runs-on: ubuntu-latest\n steps:\n # lfs: true pulls the eval corpus PDFs — they are stored via Git LFS,\n # so a default checkout would yield pointer files and the eval would fail.\n - uses: actions/checkout@v4\n with:\n lfs: true\n\n - name: Install uv\n uses: astral-sh/setup-uv@v5\n with:\n version: \"0.5.0\"\n enable-cache: true\n\n - name: Install dependencies\n run: uv sync --frozen --python 3.10\n\n # Needs GEMINI_API_KEY (generation + embedding) and the API key for\n # whichever JUDGE_PROVIDER is active. All judge keys are forwarded\n # unconditionally so a future provider swap is just a JUDGE_PROVIDER\n # change in the workflow env / repo variables — no .yml edit needed.\n # Missing keys cause the regression test to skip rather than fail.\n - name: RAGAS regression eval\n env:\n GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}\n OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}\n run: uv run pytest -m eval -v\n"}, "d0fb277fb2cdcddb5b1533fe465849ce2e1e01bc": {"t": 1785748182.7890174, "n": 1264, "h": "d9f8623e85858ef4", "c": "name: Eval Regression\n\n# Evaluation as CI: every PR runs RAGAS over a 10-question subset and the\n# job fails if faithfulness regresses. Also runnable on demand.\non:\n pull_request:\n workflow_dispatch:\n\n# A new commit on the same branch supersedes an in-flight eval run.\nconcurrency:\n group: eval-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n ragas-regression:\n runs-on: ubuntu-latest\n steps:\n # lfs: true pulls the eval corpus PDFs — they are stored via Git LFS,\n # so a default checkout would yield pointer files and the eval would fail.\n - uses: actions/checkout@v4\n with:\n lfs: true\n\n - name: Install uv\n uses: astral-sh/setup-uv@v5\n with:\n version: \"0.5.0\"\n enable-cache: true\n\n - name: Install dependencies\n run: uv sync --frozen --python 3.10\n\n # Needs GEMINI_API_KEY (generation + embedding) and OPENROUTER_API_KEY\n # (RAGAS judge) in repo secrets. Without them the test skips, so the\n # job stays green until both secrets are configured.\n - name: RAGAS regression eval\n env:\n GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}\n run: uv run pytest -m eval -v\n"}, "6bd84b8928774e74808bce5e75203d9f30461c7e": {"t": 1785748184.2815783, "n": 1125, "h": "070d2615ff5f1e9a", "c": "name: Eval Regression\n\n# Evaluation as CI: every PR runs RAGAS over a 10-question subset and the\n# job fails if faithfulness regresses. Also runnable on demand.\non:\n pull_request:\n workflow_dispatch:\n\n# A new commit on the same branch supersedes an in-flight eval run.\nconcurrency:\n group: eval-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n ragas-regression:\n runs-on: ubuntu-latest\n steps:\n # lfs: true pulls the eval corpus PDFs — they are stored via Git LFS,\n # so a default checkout would yield pointer files and the eval would fail.\n - uses: actions/checkout@v4\n with:\n lfs: true\n\n - name: Install uv\n uses: astral-sh/setup-uv@v5\n with:\n version: \"0.5.0\"\n enable-cache: true\n\n - name: Install dependencies\n run: uv sync --frozen --python 3.10\n\n # Needs GEMINI_API_KEY in repo secrets. Without it the test skips,\n # so the job stays green until the secret is configured.\n - name: RAGAS regression eval\n env:\n GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n run: uv run pytest -m eval -v\n"}, "4c1e1551b359606f948a059e511781e5f994f09c": {"t": 1785748185.3713448, "n": 934, "h": "201185a43eb9b94f", "c": "name: Eval Regression\n\n# Evaluation as CI: every PR runs RAGAS over a 10-question subset and the\n# job fails if faithfulness regresses. Also runnable on demand.\non:\n pull_request:\n workflow_dispatch:\n\n# A new commit on the same branch supersedes an in-flight eval run.\nconcurrency:\n group: eval-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n ragas-regression:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - name: Install uv\n uses: astral-sh/setup-uv@v5\n with:\n version: \"0.5.0\"\n enable-cache: true\n\n - name: Install dependencies\n run: uv sync --frozen --python 3.10\n\n # Needs GEMINI_API_KEY in repo secrets. Without it the test skips,\n # so the job stays green until the secret is configured.\n - name: RAGAS regression eval\n env:\n GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n run: uv run pytest -m eval -v\n"}}}} |