OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
42 KiB
JSON
1 line
42 KiB
JSON
{"repo": "alberthakobyan070/advanced-obsidian-rag", "files": {"src/llm/llm_client.py": {"90f083d83659cc377ec0a31dd34326f7c27a9480": {"t": 1785674012.3646433, "n": 15383, "h": "454cb9c2cc7f7b40", "c": "\"\"\"\nllm_client.py — One interface, any OpenAI- or Anthropic-compatible backend.\n\nThe rest of the pipeline NEVER imports a vendor SDK directly, so the whole\ngeneration layer swaps by editing one line in config.yaml.\n\n from src.llm.llm_client import LLMClient\n llm = LLMClient.from_config(cfg) # role=\"generation\"\n judge = LLMClient.from_config(cfg, role=\"eval.judge\")\n\nTwo ways to name a backend, and they coexist:\n\n1. PROVIDER REGISTRY (preferred). `providers:` in config.yaml holds named\n backends; a role points at one by name:\n\n providers:\n minimax:\n kind: anthropic # wire protocol, not vendor\n base_url: \"https://api.minimax.io/anthropic\"\n model: \"MiniMax-M3\"\n api_key_env: MINIMAX_API_KEY # the NAME of an env var\n api_key_prefix: \"sk-cp-\" # Token Plan key type\n generation:\n provider: minimax\n\n Keys are NEVER stored in config.yaml — a provider names the environment\n variable holding its key, and .env (gitignored) is loaded at startup. Add a\n provider for any service exposing an OpenAI-compatible /v1/chat/completions\n or an Anthropic-compatible endpoint: MiniMax, Moonshot/Kimi, xAI, OpenAI,\n a local KoboldCPP, an office proxy. `kind` selects the wire protocol only.\n\n2. LEGACY (unchanged, still fully supported). The three names `openai`,\n `anthropic` and `local` are RESERVED and always take the original code\n path with their original env vars (OPENAI_API_KEY / ANTHROPIC_API_KEY).\n Never add a registry entry under those names — existing configs, including\n the Docker bundle, depend on them resolving the old way.\n\nPer-role: any config section may carry `provider:` / `temperature:` /\n`max_tokens:`, so generation, HyDE and the eval judge can each run on a\ndifferent backend. `<role>.model` overrides the provider's default model.\n\nLocal note (KoboldCPP serving Gemma-4-E4B-it-Q8):\n KoboldCPP exposes an OpenAI-compatible API at /v1, so we reuse the OpenAI\n SDK and just point base_url at it with a dummy key. Context length must be\n set when LAUNCHING KoboldCPP (--contextsize N); it cannot be set per-request.\n\"\"\"\nfrom __future__ import annotations\n\nfrom dataclasses import dataclass\nfrom typing import Any\n\nfrom src.utils.config_loader import Config\nfrom src.utils.logger import get_logger\n\nlog = get_logger(__name__)\n\n\n@dataclass\nclass LLMResponse:\n text: str\n model: str\n provider: str\n usage: dict[str, Any] | None = None\n\n\nclass LLMClient:\n \"\"\"Provider-agnostic chat completion wrapper.\"\"\"\n\n def __init__(\n self,\n provider: str,\n model: str,\n api_key: str | None = None,\n base_url: str | None = None,\n default_temperature: float = 0.1,\n default_max_tokens: int = 1500,\n backend: str | None = None,\n ):\n # `provider` is the wire protocol used by complete(); `backend` is the\n # configured registry alias (minimax, freellmapi, ...). Keeping both\n # prevents per-request backend comparisons from collapsing every\n # OpenAI-compatible service into the label \"openai\".\n self.provider = provider\n self.backend = backend or provider\n self.model = model\n self.default_temperature = default_temperature\n self.default_max_tokens = default_max_tokens\n self._client = self._build_client(provider, api_key, base_url)\n log.info(\n \"LLMClient ready: backend=%s provider=%s model=%s\",\n self.backend, provider, model,\n )\n\n # ---- construction -------------------------------------------------\n\n # Names that always take the legacy path below. A registry entry under one\n # of these would silently redirect every pre-registry config.\n RESERVED_PROVIDERS = (\"openai\", \"anthropic\", \"local\")\n\n @classmethod\n def from_config(cls, cfg: Config, role: str = \"generation\") -> \"LLMClient\":\n \"\"\"\n Build a client for a config section: \"generation\", \"eval.judge\", ...\n\n Resolution order: a `providers:` registry entry whose name matches\n `<role>.provider` wins; otherwise the reserved legacy names apply.\n \"\"\"\n provider = cfg.get(f\"{role}.provider\", \"anthropic\")\n temperature = cfg.get(f\"{role}.temperature\", 0.1)\n max_tokens = cfg.get(f\"{role}.max_tokens\", 1500)\n\n registry = cfg.get(\"providers\", {}) or {}\n if provider in registry and provider not in cls.RESERVED_PROVIDERS:\n return cls._from_registry(\n cfg, role, provider, registry[provider], temperature, max_tokens\n )\n if provider not in cls.RESERVED_PROVIDERS and registry:\n raise ValueError(\n f\"{role}.provider = {provider!r} is not in the `providers:` \"\n f\"registry and is not one of {cls.RESERVED_PROVIDERS}. \"\n f\"Known providers: {sorted(registry)}\"\n )\n\n if provider == \"anthropic\":\n return cls(\n provider=\"anthropic\",\n model=cfg.get(f\"{role}.model\", \"claude-sonnet-4-20250514\"),\n api_key=cfg.require_secret(\"ANTHROPIC_API_KEY\"),\n default_temperature=temperature,\n default_max_tokens=max_tokens,\n )\n if provider == \"openai\":\n return cls(\n provider=\"openai\",\n model=cfg.get(f\"{role}.model\", \"gpt-4o-mini\"),\n api_key=cfg.require_secret(\"OPENAI_API_KEY\"),\n base_url=cfg.get(f\"{role}.base_url\"),\n default_temperature=temperature,\n default_max_tokens=max_tokens,\n )\n if provider == \"local\":\n return cls(\n provider=\"local\",\n model=cfg.get(f\"{role}.local.model\", \"local-model\"),\n api_key=cfg.get(f\"{role}.local.api_key\", \"local\"),\n base_url=cfg.get(f\"{role}.local.base_url\", \"http://localhost:5001/v1\"),\n default_temperature=temperature,\n default_max_tokens=max_tokens,\n )\n raise ValueError(f\"Unknown LLM provider: {provider!r}\")\n\n @classmethod\n def from_provider_override(\n cls,\n cfg: Config,\n provider: str,\n model: str | None = None,\n role: str = \"generation\",\n ) -> \"LLMClient\":\n \"\"\"Build a request-local client for a configured backend.\n\n Registry backends use their own default model, not ``<role>.model``:\n that role value belongs to the process-wide default backend and may be\n invalid for the requested one. ``model`` is the only per-request model\n override. The supplied ``Config`` is read only and is never copied or\n mutated.\n\n Reserved legacy providers remain available and retain their established\n construction rules. For them, ``model`` overrides the configured role\n model when supplied.\n \"\"\"\n temperature = cfg.get(f\"{role}.temperature\", 0.1)\n max_tokens = cfg.get(f\"{role}.max_tokens\", 1500)\n registry = cfg.get(\"providers\", {}) or {}\n\n if provider in registry and provider not in cls.RESERVED_PROVIDERS:\n return cls._from_registry(\n cfg,\n role,\n provider,\n registry[provider],\n temperature,\n max_tokens,\n model_override=model,\n inherit_role_model=False,\n )\n if provider not in cls.RESERVED_PROVIDERS and registry:\n raise ValueError(\n f\"{role}.provider = {provider!r} is not in the `providers:` \"\n f\"registry and is not one of {cls.RESERVED_PROVIDERS}. \"\n f\"Known providers: {sorted(registry)}\"\n )\n\n if provider == \"anthropic\":\n return cls(\n provider=\"anthropic\",\n model=model or cfg.get(\n f\"{role}.model\", \"claude-sonnet-4-20250514\"\n ),\n api_key=cfg.require_secret(\"ANTHROPIC_API_KEY\"),\n default_temperature=temperature,\n default_max_tokens=max_tokens,\n backend=provider,\n )\n if provider == \"openai\":\n return cls(\n provider=\"openai\",\n model=model or cfg.get(f\"{role}.model\", \"gpt-4o-mini\"),\n api_key=cfg.require_secret(\"OPENAI_API_KEY\"),\n base_url=cfg.get(f\"{role}.base_url\"),\n default_temperature=temperature,\n default_max_tokens=max_tokens,\n backend=provider,\n )\n if provider == \"local\":\n return cls(\n provider=\"local\",\n model=model or cfg.get(f\"{role}.local.model\", \"local-model\"),\n api_key=cfg.get(f\"{role}.local.api_key\", \"local\"),\n base_url=cfg.get(\n f\"{role}.local.base_url\", \"http://localhost:5001/v1\"\n ),\n default_temperature=temperature,\n default_max_tokens=max_tokens,\n backend=provider,\n )\n raise ValueError(f\"Unknown LLM provider: {provider!r}\")\n\n @classmethod\n def _from_registry(\n cls,\n cfg: Config,\n role: str,\n name: str,\n spec: dict,\n temperature: float,\n max_tokens: int,\n model_override: str | None = None,\n inherit_role_model: bool = True,\n ) -> \"LLMClient\":\n \"\"\"Build a client from a `providers:` entry.\n\n The provider owns the backend facts (endpoint, model, which env var\n holds its key); the role may override `model` to run a different model\n on the same backend. That override is logged, because pointing a role\n at a new provider while leaving a stale `<role>.model` behind is the\n one way to misconfigure this quietly — the model name simply gets\n rejected by the new endpoint.\n \"\"\"\n spec = spec or {}\n kind = str(spec.get(\"kind\", \"openai\")).lower()\n if kind not in (\"openai\", \"anthropic\"):\n raise ValueError(\n f\"providers.{name}.kind = {kind!r}; expected 'openai' or \"\n f\"'anthropic' (the wire protocol, not the vendor)\"\n )\n\n spec_model = spec.get(\"model\")\n role_model = (\n cfg.get(f\"{role}.model\") if inherit_role_model else model_override\n )\n model = role_model or spec_model\n if not model:\n raise ValueError(\n f\"No model for role {role!r}: set providers.{name}.model \"\n f\"or {role}.model\"\n )\n\n # Keys come from the environment, never from config.yaml. A provider\n # with no api_key_env is a keyless local endpoint (KoboldCPP, Ollama,\n # a localhost proxy) — the OpenAI SDK still wants a non-empty string.\n key_env = spec.get(\"api_key_env\")\n if key_env:\n raw_key = (cfg.secret(key_env) if spec.get(\"api_key_optional\")\n else cfg.require_secret(key_env))\n expected_prefix = str(spec.get(\"api_key_prefix\") or \"\")\n if (raw_key and expected_prefix\n and not raw_key.startswith(expected_prefix)):\n raise RuntimeError(\n f\"{key_env} has the wrong credential type for \"\n f\"providers.{name}; expected a key beginning with \"\n f\"{expected_prefix!r}\"\n )\n api_key = raw_key or \"not-needed\"\n else:\n api_key = \"not-needed\"\n\n if role_model and spec_model and role_model != spec_model:\n log.warning(\n \"%s.model=%r overrides providers.%s.model=%r — make sure %s \"\n \"actually serves that model\",\n role, role_model, name, spec_model, name,\n )\n log.info(\"LLMClient[%s]: provider=%s kind=%s model=%s\", role, name, kind, model)\n\n return cls(\n provider=kind,\n model=model,\n api_key=api_key,\n base_url=spec.get(\"base_url\"),\n default_temperature=temperature,\n default_max_tokens=max_tokens,\n backend=name,\n )\n\n def _build_client(self, provider: str, api_key, base_url):\n if provider == \"anthropic\":\n try:\n import anthropic\n except ImportError as e:\n raise ImportError(\"pip install anthropic\") from e\n # base_url matters here too: several providers (MiniMax among them)\n # expose an Anthropic-compatible endpoint alongside their /v1 one.\n kwargs: dict[str, Any] = {\"api_key\": api_key}\n if base_url:\n kwargs[\"base_url\"] = base_url\n return anthropic.Anthropic(**kwargs)\n # openai + local both use the OpenAI SDK\n try:\n from openai import OpenAI\n except ImportError as e:\n raise ImportError(\"pip install openai\") from e\n kwargs: dict[str, Any] = {\"api_key\": api_key}\n if base_url:\n kwargs[\"base_url\"] = base_url\n return OpenAI(**kwargs)\n\n # ---- completion ---------------------------------------------------\n\n def complete(\n self,\n system: str,\n user: str,\n temperature: float | None = None,\n max_tokens: int | None = None,\n ) -> LLMResponse:\n \"\"\"Single-turn system+user completion. Synchronous.\"\"\"\n temp = self.default_temperature if temperature is None else temperature\n maxtok = self.default_max_tokens if max_tokens is None else max_tokens\n\n if self.provider == \"anthropic\":\n return self._complete_anthropic(system, user, temp, maxtok)\n return self._complete_openai_compatible(system, user, temp, maxtok)\n\n def _complete_anthropic(self, system, user, temp, maxtok) -> LLMResponse:\n resp = self._client.messages.create(\n model=self.model,\n system=system,\n max_tokens=maxtok,\n temperature=temp,\n messages=[{\"role\": \"user\", \"content\": user}],\n )\n text = \"\".join(\n block.text for block in resp.content if getattr(block, \"type\", \"\") == \"text\"\n )\n usage = None\n if getattr(resp, \"usage\", None):\n usage = {\n \"input_tokens\": resp.usage.input_tokens,\n \"output_tokens\": resp.usage.output_tokens,\n }\n return LLMResponse(text=text, model=self.model, provider=self.provider, usage=usage)\n\n def _complete_openai_compatible(self, system, user, temp, maxtok) -> LLMResponse:\n resp = self._client.chat.completions.create(\n model=self.model,\n temperature=temp,\n max_tokens=maxtok,\n messages=[\n {\"role\": \"system\", \"content\": system},\n {\"role\": \"user\", \"content\": user},\n ],\n )\n text = resp.choices[0].message.content or \"\"\n usage = None\n if getattr(resp, \"usage\", None):\n usage = {\n \"input_tokens\": resp.usage.prompt_tokens,\n \"output_tokens\": resp.usage.completion_tokens,\n }\n return LLMResponse(text=text, model=self.model, provider=self.provider, usage=usage)\n"}}, "config.example.yaml": {"90f083d83659cc377ec0a31dd34326f7c27a9480": {"t": 1786068426.3056693, "n": 26174, "h": "225503b056ed57ad", "c": "# ============================================================\n# Personal RAG — central configuration\n# Copy this file to config.yaml and adapt the paths to your machine.\n# config.yaml is gitignored, so pulls never overwrite your settings.\n# Secrets live in .env (also gitignored). Tunables live here.\n# ============================================================\n\nparser:\n # Any string value in this file may reference an environment variable as\n # ${NAME} or ${NAME:-default} — useful for containers, where the vault path\n # has to match a bind-mount. An unset ${NAME} stays literal rather than\n # becoming \"\", which would point the parser at the filesystem root.\n vault_path: \"path/to/your/documents\"\n max_chunk_size: 3000\n min_chunk_size: 200\n overlap_size: 150\n chunking: heading # heading | fixed | document | none (how OVERSIZED sections split)\n skip_dirs: [\".obsidian\", \".trash\", \".git\", \"__path_fix_backups\", \"__MACOSX\"]\n\n# ---------------------------------------------------------------------------\n# What the folder-derived grouping label is CALLED, and where it comes from.\n#\n# The METADATA keys are always course_code / course_name whatever `label` says.\n# Renaming them would invalidate every stored chunk and force a full re-embed,\n# so only the console's wording and the folder maps are configurable.\n#\n# The shipped parser has built-in maps for an academic vault (course folders,\n# CS/DS course codes). They are a DEFAULT, not a requirement:\n# * detect_from_path: false -> nothing is derived from folder names; every\n# chunk is \"unknown\" until you retag. Right for\n# a vault with no folder convention to mine.\n# * all four maps null -> keep the built-in academic maps.\n# * ANY map supplied -> the built-ins are dropped WHOLESALE and only\n# what you supply applies. All-or-nothing, so a\n# vault that declares its own taxonomy can't\n# still be labelled through the built-in\n# keyword fallback.\n# ---------------------------------------------------------------------------\ntaxonomy:\n label: course # e.g. course | project | client | topic\n label_plural: courses\n detect_from_path: true\n folder_map: null # {\"folder name\": \"Label\"}\n domain_map: null # {\"Label\": \"ml\"} (stats/ml/nlp/viz/biz/ts/db/math/swe/prog)\n code_map: null # {\"CS 101\": \"Label\"}\n keywords: null # {\"substring\": \"Label\"} — last-resort match\n\npaths:\n chunks_file: \"data/chunks.jsonl\"\n chroma_dir: \"data/chroma_db\"\n bm25_index: \"data/bm25_index.pkl\"\n collection_name: \"obsidian_vault\"\n\n\nembedding:\n # provider: openai | local\n # openai -> text-embedding-3-small via API\n # local -> sentence-transformers model running on this machine (no API cost)\n provider: local\n local_model: \"BAAI/bge-small-en-v1.5\"\n batch_size: 100\n dimensions: 384\n\n# ---------------------------------------------------------------------------\n# Generation backends. `kind` is the WIRE PROTOCOL (openai | anthropic), not\n# the vendor — anything that speaks one of those two drops in here.\n#\n# Three names are RESERVED and always selectable without an entry: openai,\n# anthropic, local. Never add an entry under one of those.\n#\n# A key is NEVER written here. An entry names the environment variable that\n# holds it (`api_key_env`); the console's Settings tab writes that variable\n# into the gitignored .env and reports only whether it is set.\n#\n# MODEL IDS MOVE. The ones below were the current default for each endpoint\n# when this file was written and are only a starting point — every endpoint\n# lists what it actually serves at GET <base_url>/models, and the console\n# copies an entry's `model` into generation.model when you activate it.\n#\n# Nothing here costs anything or runs until you select it as\n# generation.provider AND its key is present.\n# ---------------------------------------------------------------------------\nproviders:\n freellmapi:\n label: \"FreeLLMAPI proxy\"\n description: \"Local proxy that fronts free-tier providers; keys live in its own dashboard.\"\n kind: openai\n base_url: \"http://localhost:3001/v1\"\n model: \"auto\"\n api_key_env: FREELLMAPI_API_KEY\n api_key_optional: true\n\n # ---- hosted, OpenAI-compatible ----\n openrouter:\n label: \"OpenRouter\"\n description: \"One key, most models. `openrouter/auto` picks per request.\"\n kind: openai\n base_url: \"https://openrouter.ai/api/v1\"\n model: \"openrouter/auto\"\n api_key_env: OPENROUTER_API_KEY\n groq:\n label: \"Groq\"\n description: \"Open-weight models on custom silicon; very low latency.\"\n kind: openai\n base_url: \"https://api.groq.com/openai/v1\"\n model: \"llama-3.3-70b-versatile\"\n api_key_env: GROQ_API_KEY\n deepseek:\n label: \"DeepSeek\"\n kind: openai\n base_url: \"https://api.deepseek.com/v1\"\n model: \"deepseek-chat\"\n api_key_env: DEEPSEEK_API_KEY\n mistral:\n label: \"Mistral\"\n kind: openai\n base_url: \"https://api.mistral.ai/v1\"\n model: \"mistral-large-latest\"\n api_key_env: MISTRAL_API_KEY\n nvidia_nim:\n label: \"NVIDIA NIM\"\n description: \"NVIDIA-hosted inference for open-weight models.\"\n kind: openai\n base_url: \"https://integrate.api.nvidia.com/v1\"\n model: \"meta/llama-3.3-70b-instruct\"\n api_key_env: NVIDIA_API_KEY\n together:\n label: \"Together AI\"\n kind: openai\n base_url: \"https://api.together.xyz/v1\"\n model: \"meta-llama/Llama-3.3-70B-Instruct-Turbo\"\n api_key_env: TOGETHER_API_KEY\n kimi:\n label: \"Moonshot / Kimi\"\n kind: openai\n base_url: \"https://api.moonshot.ai/v1\"\n model: \"kimi-k2-0711-preview\"\n api_key_env: MOONSHOT_API_KEY\n grok:\n label: \"xAI / Grok\"\n description: \"Needs an x.ai API key — an X Premium subscription is not one.\"\n kind: openai\n base_url: \"https://api.x.ai/v1\"\n model: \"grok-4\"\n api_key_env: XAI_API_KEY\n # MiniMax Token Plan is an API subscription: paste its sk-cp- Subscription\n # Key into MINIMAX_API_KEY. A pay-as-you-go sk-api key is a different\n # credential and will not consume the plan's quota — the prefix check below\n # rejects it rather than letting it bill separately by surprise.\n minimax:\n label: \"MiniMax M3 · Token Plan\"\n description: \"Subscription/Credits key from Billing > Token Plan\"\n kind: anthropic\n base_url: \"https://api.minimax.io/anthropic\"\n model: \"MiniMax-M3\"\n api_key_env: MINIMAX_API_KEY\n api_key_prefix: \"sk-cp-\"\n\n # ---- models you run yourself (no key, nothing leaves the machine) ----\n # `model` must name something the server actually has loaded; ask it with\n # curl <base_url>/models\n ollama:\n label: \"Ollama (local)\"\n kind: openai\n base_url: \"http://localhost:11434/v1\"\n model: \"llama3.1:8b\"\n api_key_env: null\n lmstudio:\n label: \"LM Studio (local)\"\n kind: openai\n base_url: \"http://localhost:1234/v1\"\n model: \"local-model\"\n api_key_env: null\n kobold:\n label: \"KoboldCPP (local)\"\n kind: openai\n base_url: \"http://localhost:5001/v1\"\n model: \"local-model\"\n api_key_env: null\n\ngeneration:\n # A `providers:` name or a reserved legacy name (openai | anthropic | local).\n # Change `model` whenever you change this.\n provider: freellmapi\n base_url: \"http://localhost:3001/v1\" # legacy providers only\n model: \"auto\"\n temperature: 0.1\n max_tokens: 2500\n verify_citations: false # text-embedding-3-small native dim\n # Reserved legacy `local` provider (KoboldCPP/OpenAI-compatible). Prefer a\n # named registry entry for new setups; this block preserves older configs.\n local:\n base_url: \"http://localhost:5001/v1\"\n api_key: \"koboldcpp\" # dummy; KoboldCPP ignores it\n model: \"gemma-4-E4B-it-Q8_k\" # whatever /v1/models reports\n # Context length is set when KoboldCPP launches, not per request.\n\nretrieval:\n dense_top_k: 20\n sparse_top_k: 20\n rrf_k: 60\n rerank_top_k: 7\n rerank_mode: cross_encoder # cross_encoder | http | lexical | none\n # Any HF cross-encoder id. Bigger rerankers score better on public benchmarks\n # but cost far more per query — measure before switching:\n # cross-encoder/ms-marco-MiniLM-L-6-v2 22M params fast (default)\n # BAAI/bge-reranker-base 278M params medium\n # BAAI/bge-reranker-v2-m3 568M params slow on CPU (~20x),\n # multilingual, 8k context, GPU\n # strongly recommended\n cross_encoder_model: \"cross-encoder/ms-marco-MiniLM-L-6-v2\"\n cross_encoder_max_length: 512 # bge-base hard limit; v2-m3 supports 8192\n cross_encoder_device: auto # auto | cpu | cuda | cuda:0 | cuda:1\n # Used only when rerank_mode: http. Point at any OpenAI-style /v1/rerank\n # service (llama.cpp --reranking, TEI, Infinity, etc.).\n rerank_http:\n base_url: \"http://127.0.0.1:8101/v1\"\n model: \"bge-reranker-v2-m3\"\n timeout: 120\n # Ranking CRITERION for the reranker, in plain language. Retrieval already\n # built the candidate pool against the real question; this only changes the\n # text the RERANKER scores against, so it reweights results and can never\n # remove any. Blank = off.\n # e.g. \"prefer worked procedures and runnable examples over definitions\"\n # \"prefer primary sources over summaries of them\"\n # Applies to rerank_mode cross_encoder and http. IGNORED by lexical (it would\n # dilute the query's own terms) and by none. Per-call override:\n # {\"rerank_instruction\": \"...\"} on /search and /query.\n rerank_instruction: \"\"\n # prefix = the instruction, a newline, then the question. Works with\n # any cross-encoder: it simply reframes the text scored.\n # instruct = the labelled <Instruct>/<Query> form. Only for a reranker\n # documented to expect it.\n rerank_instruction_format: prefix\n use_hyde: true\n # If a query mentions a course/domain, bias retrieval toward it\n metadata_boost: true\n\n # Small-to-big context expansion (post-rerank), both OFF by default; the\n # `synthesis` preset turns them on. parent_context swaps a markdown chunk for\n # its whole enclosing section, read from the sidecar below (parents are never\n # embedded). neighbor_context appends adjacent pages of the same PDF as\n # marked supplementary sources.\n parent_context: false\n parents_file: \"data/parents_md.jsonl\" # per-corpus; the vault switcher moves it\n neighbor_context: false\n neighbor_max_per_doc: 2\n neighbor_max_total: 4\n neighbor_max_chars: 2500\n\n # HyDE bypass for code-intent queries. If the query contains any of these\n # signals (case-insensitive, word-boundary matched; trailing '_' or '.'\n # behaves like a prefix, e.g. \"geom_\" matches geom_point), the hypothetical-\n # answer LLM call is skipped and the raw query is used for search. Prose\n # hypotheticals bias dense retrieval toward lecture notes and away from\n # actual code chunks (~2.4% of the corpus). Extend freely — no code change.\n hyde_skip_signals:\n - code\n - script\n - function\n - snippet\n - notebook\n - implement\n - implementation\n - \"show me my\"\n - \"my r\"\n - \"my python\"\n - ggplot\n - geom_\n - dplyr\n - tidyverse\n - matplotlib\n - seaborn\n - pandas\n - numpy\n - sklearn\n - pytorch\n - regex\n - sql\n\n # File types that count as \"the author's own code\" for the code lane: when a\n # code-intent query fires, an extra dense+sparse pass restricted to these\n # types joins the RRF fusion, guaranteeing actual script/notebook chunks\n # reach the reranker (they're ~2.4% of the corpus and otherwise get\n # outscored by lecture PDFs that merely mention the same keywords).\n code_file_types: [ipynb, py, r, rmd]\n\n # ------------------------------------------------------------------\n # Scope routing: queries that NAME a domain or content type get extra\n # filtered dense+sparse lanes in the RRF fusion (soft routing — the\n # reranker still decides; a bad hint can't empty the results).\n # Same matcher as hyde_skip_signals: word-boundary, plural-tolerant.\n # ------------------------------------------------------------------\n\n # domain metadata value -> query keywords/aliases that point at it.\n # Keys MUST be values the `domain` metadata field actually holds\n # (stats, ml, nlp, viz, biz, ts, db, math, swe, prog).\n domain_signals:\n stats: [statistics, statistical, stats, probability, bayesian, bayes,\n hypothesis testing, conjugate prior, confidence interval]\n ml: [machine learning, ml, deep learning, dl, neural network,\n neural networks, sklearn, scikit-learn, pytorch, tensorflow,\n reinforcement learning, rl, genai, generative ai, llm,\n gradient descent, hugging face]\n nlp: [nlp, natural language processing, natural language, transformer,\n transformers, bert, mbert, tokenizer, tokenization, embedding,\n embeddings, spacy, nltk, stanza, sentencepiece]\n viz: [visualization, visualizations, dataviz, data viz,\n data visualization, ggplot, matplotlib, seaborn, plotly,\n tableau, power bi]\n biz: [business intelligence, bi, business analytics,\n marketing analytics, marketing, dax, power bi, tableau]\n ts: [time series, forecasting, arima, sarima, seasonality]\n db: [database, databases, sql, postgresql, postgres, mysql, mongodb,\n data engineering, sqlalchemy]\n math: [calculus, linear algebra, numerical methods, math, mathematics,\n eigenvalue, eigenvalues]\n swe: [software architecture, software engineering, devops, docker,\n kubernetes, fastapi, web development, design pattern,\n design patterns, microservices]\n prog: [data structures, algorithms, python development]\n\n # content label -> {signals: query keywords, path_contains: source_file\n # substrings (case-insensitive), file_types: file_type metadata values}.\n # Content aspects OR together; domain AND content must both hold\n # (\"my homework from NLP\" = homework path AND nlp domain).\n content_signals:\n homework: {signals: [homework, hw, assignment, assignments, problem set],\n path_contains: [homework, hw, assignment]}\n lectures: {signals: [lecture, lectures, slides, lecture notes],\n path_contains: [lecture, slides]}\n tech_books: {signals: [tech book, tech books],\n path_contains: [tech books]}\n books: {signals: [textbook, textbooks, book, books],\n path_contains: [books]}\n cheatsheets: {signals: [cheat sheet, cheat sheets, cheatsheet, cheatsheets],\n path_contains: [cheat sheet, cheatsheet]}\n notebooks: {signals: [notebook, notebooks, jupyter],\n file_types: [ipynb]}\n capstone: {signals: [capstone, thesis],\n path_contains: [capstone]}\n coursera: {signals: [coursera],\n path_contains: [coursera]}\n daily_notes: {signals: [daily note, daily notes, study notes],\n file_types: [daily_note]}\n current: {signals: [current course, current courses, this semester],\n path_contains: [current courses]}\n passed: {signals: [passed course, passed courses, passed coursework],\n path_contains: [passed]}\n\n# ------------------------------------------------------------------\n # Live-vault lane via Obsidian's Omnisearch plugin (community plugin).\n # Adds an extra RRF lane of LIVE results — notes edited since the last\n # ingest, with Omnisearch's filename/heading-weighted BM25. Requires\n # Obsidian OPEN with Omnisearch's \"HTTP Server\" setting enabled\n # (default port 51361, localhost only). Fail-soft: Obsidian closed ->\n # the lane is silently empty, queries never break. Toggle per query\n # with {\"omnisearch\": true/false} on /query and /search.\n # ------------------------------------------------------------------\n omnisearch:\n enabled: false # default OFF; flip on, or pass per-query\n base_url: \"http://127.0.0.1:51361\"\n top_k: 8 # live results offered to the fusion\n timeout: 1.5 # seconds; keeps a closed Obsidian cheap\n excerpt_min_chars: 40 # drop excerpts too short to rerank meaningfully\n # allowed_exts: [.md, .markdown, .txt, .canvas] # PDFs/images skipped —\n # they're in our own index already\n\n # Named retrieval presets — per-query bundles of overrides, selected via\n # POST /query {\"preset\": \"code\"}, `python main.py query --preset code`, or\n # the Streamlit sidebar. The \"code\" preset is auto-applied when a query\n # trips hyde_skip_signals (wider candidate pool so BM25's code hits survive\n # RRF fusion and reach the reranker). Omitted keys fall back to the\n # defaults above.\n # (presets may also set `omnisearch: true` to include the live-vault lane)\n presets:\n code: {rerank_top_k: 10, use_hyde: false, dense_top_k: 40, sparse_top_k: 40, boost_code: true}\n concept: {rerank_top_k: 5, use_hyde: true}\n synthesis: {rerank_top_k: 10, use_hyde: true, dense_top_k: 30, sparse_top_k: 30}\n fresh: {rerank_top_k: 8, omnisearch: true}\n\n\n# Golden-query eval suite (eval/eval_runner.py — `python main.py eval`).\n# Tier 1 (retrieval) and the groundedness floor need nothing but the index;\n# the judge tier reuses the `generation` client above.\neval:\n ngram_n: 2 # content-word n-gram size for the deterministic\n # groundedness floor. 1 = lenient (bag of words),\n # 3 = strict (near-verbatim). 2 is the balance.\n judge:\n # Leave empty to reuse the generation client (self-grading — the runner\n # warns). Name a `providers:` entry to judge with a different backend.\n provider: null\n model: null\n temperature: 0.0\n max_tokens: 700 # per LLM-as-judge call (rubric JSON is small)\n\nlogging:\n level: INFO # DEBUG | INFO | WARNING | ERROR\n file: \"logs/rag.log\"\n console: true\n\npdf:\n output_file: \"data/pdf_chunks.jsonl\"\n scratch_dir: null # optional: route HF cache + temp to a big drive\n min_chunk_size: 200\n max_chunk_size: 3000\n overlap_size: 150\n skip_scanned_threshold: 50 # chars/page below which a page = scanned\n skip_list_file: null # optional JSON array of vault-relative paths to skip\n skip_scripts: [\"armenian\"]\n script_sample_pages: 3\n skip_folders: []\n extract_images: true\n figures_dir: data/figures\n image_min_frac: 0.08\n ocr_enabled: true\n ocr_language: \"eng\" # add \"+hye\" for Armenian if tessdata present\n # OCR engine for scanned pages:\n # auto -> historical behavior (Tesseract-first probe) [default]\n # tesseract -> pin the classic path\n # vlm -> vision-model document parsing via the endpoint below\n # none -> disable OCR (same as --no-ocr)\n ocr_engine: auto\n # VLM document-parsing OCR (reference model: baidu/Unlimited-OCR, 3B).\n # Reached over ANY OpenAI-compatible /chat/completions endpoint that takes\n # image_url content — SGLang/vLLM on a GPU box, llama.cpp/LM Studio serving\n # a GGUF quant locally, or FreeLLMAPI (:3001) fronting a free hosted vision\n # model. Per-page markdown replaces the scanned page's empty extraction.\n # Named VLM-OCR engines. Each model needs its OWN prompt and dpi —\n # DeepSeek-OCR requires llama.cpp's <__media__> marker because it ships no\n # chat template of its own, the others don't — so a preset bundles them and\n # switching engine is one word instead of four keys that fail confusingly\n # when mismatched. Selected by `vlm_ocr.preset` below.\n vlm_ocr_presets:\n deepseek:\n model: \"DeepSeek-OCR\"\n prompt: \"<__media__>\\nConvert the document to markdown.\"\n dpi: 150\n paddleocr_vl:\n model: \"PaddleOCR-VL\"\n prompt: \"Convert the document to markdown.\"\n dpi: 200\n glm_ocr:\n model: \"GLM-OCR\"\n prompt: \"Convert the document to markdown.\"\n dpi: 200\n\n vlm_ocr:\n preset: null # a name above, or null for the keys here\n base_url: \"http://127.0.0.1:8100/v1\"\n model: \"Unlimited-OCR\" # match what the server reports\n api_key_env: \"OPENAI_API_KEY\" # sent if set; local servers ignore it\n prompt: \"<image>document parsing.\" # Unlimited-OCR's parsing prompt\n dpi: 200 # 300 for dense math scans (slower)\n timeout: 300 # generous — CPU-quant servers are slow\n max_tokens: 8192\n max_pages_per_pdf: null # safety cap for huge scanned books\n # Free text shown verbatim in the console's OCR panel when the endpoint is\n # down — your own reminder of how to start it. Never executed.\n launch_hint: \"Start your vision server on :8100, then press Probe.\"\n # PaddleOCR SIDECAR — the middle lane between Tesseract and a vision model.\n # Real detection+recognition (far better on skewed, multi-column and\n # non-Latin scans than Tesseract) at seconds per page on CPU. It runs in its\n # OWN container/process, so its heavy dependency tree never enters this\n # environment and you can stop it to reclaim RAM between ingests.\n # Turn it on with pdf.ocr_engine: paddle. Delete this block to disable the\n # lane entirely (the console then reports it as \"not configured\").\n # WHICH PADDLE ENGINE. The sidecar serves two, and they are different\n # products sharing a package name:\n #\n # ocr PP-OCRv6 detect-then-recognise. Plain text, one line per detected\n # line. This is what \"PaddleOCR\" has always meant here.\n # vl PaddleOCR-VL-1.6-0.9B. Layout detection plus a document VLM that\n # emits MARKDOWN: reflowed paragraphs, `#` headings, `$...$` LaTeX for\n # formulas, real tables, and running headers/footers dropped.\n #\n # Benchmarked over 7 dense textbook pages scored against each PDF's own text\n # layer, on a Pascal-generation 8GB card. See ocr-sidecar/README.md for the\n # full table. The short version:\n #\n # PP-OCRv6 ~3s/page 0.964 mean word recall\n # VL ~91s/page 0.933 mean word recall, ~7.9GB VRAM\n #\n # So `ocr` stays the default: ~30x cheaper AND slightly more accurate on\n # prose. Choose `vl` per-ingest when the STRUCTURE is the point — a chapter\n # that is mostly equations or tables, where VL reconstructs a table as real\n # HTML while PP-OCRv6 returns its digits with every relationship gone. On a\n # 400-page book that choice is ~20 minutes versus ~10 hours; do the\n # arithmetic before setting it globally. A faster card changes the ratio,\n # not the shape of the trade.\n #\n # `vl` also needs the paddlex[ocr] extras, which only the GPU image installs.\n # A sidecar that cannot serve it answers 501 rather than quietly returning\n # PP-OCRv6 output under the other engine's name.\n #\n # null = send nothing and let the sidecar's PADDLE_OCR_PIPELINE decide.\n paddle_ocr:\n base_url: \"http://127.0.0.1:8103\"\n lang: \"en\" # server-side PaddleOCR language code\n pipeline: null # null | ocr | vl (see above)\n dpi: 200 # 300 for dense small print (slower)\n timeout: 120 # RAISE to ~300 for pipeline: vl\n max_pages_per_pdf: null # safety cap for huge scanned books\n max_edge_px: 2000 # cap the rendered page's long edge\n only_book_folders: false # true = only ingest PDFs under Books/ dirs\n max_pages_per_pdf: null # cap pages (null = no cap)\nnotebooks:\n # vault_path falls back to pdf.vault_path -> parser.vault_path if unset\n output_file: \"data/ipynb_chunks.jsonl\"\n extensions: [\".ipynb\", \".py\", \".R\", \".Rmd\"] # file types to ingest\n min_chunk_size: 200\n max_chunk_size: 3000\n overlap_size: 150\n include_outputs: true # append text cell outputs\n max_output_chars: 2000 # cap per-cell output length\n skip_scripts: [\"armenian\"] # per-cell: placeholder Armenian prose/output\n save_figures: false # true = decode image outputs to figures_dir\n figures_dir: data/notebook_figures\n\nwebui:\n # Management console (manage_api.py) — ingest/index/OCR/delete from a browser.\n # python -m uvicorn manage_api:app --host 127.0.0.1 --port 8052\n port: 8052 # 8051 = query API, 8501 = Streamlit\n rag_api: \"http://127.0.0.1:8051\" # the query endpoint the console talks to\n auto_restart_rag: false # restart :8051 automatically after index-changing jobs\n # Browser uploads land INSIDE the vault here (created if missing), so the\n # normal ingest passes pick them up with --include-path \"Inbox\":\n inbox_dir: \"Inbox\"\n # Vault-tab browse root, vault-relative. \"\" = the vault root. Set it to a\n # subfolder if only part of your vault is material worth indexing.\n vault_tree_root: \"\"\n jobs_dir: \"logs/jobs\" # one log file per job\n manifest_cache: \"data/.manifest_cache.json\"\n # NOTE: ingest jobs started from the console write their output JSONL next to\n # paths.chunks_file (the ACTIVE vault's data dir), not into the project's\n # ./data — otherwise switching vaults leaves new chunks in the old vault's\n # folder, where the next BM25 rebuild sweeps them into the wrong corpus.\n\n# NOTE: the vault switcher (Settings → Vaults) carries all of these PER VAULT —\n# vault_path, the index trio, collection name, inbox_dir, vault_tree_root and\n# manifest_cache travel together, so a second vault never inherits the first\n# one's browse root or cached in-RAG status. Every key it manages must EXIST in\n# this file, or a switch fails with \"found 0 matches\".\n"}}}} |