{"repo": "chadananda/site2rag", "files": {"src/slp-context.js": {"4e012e5efbf018ec6e17a12e31d1cb3e1ae898d7": {"t": 1786068266.847429, "n": 4794, "h": "67c249fe1abb0488", "c": "// Builds the free-text `context` block sent with an SLP job. Distills the document's hosting\n// page(s) — title, link text, description, author, language, subject, likely proper nouns — via a\n// cheap DeepSeek call. SLP feeds this into OCR synthesis + metadata extraction (better proper-noun\n// and archaic-spelling resolution). Cache-optimized: the system prompt is byte-stable across calls\n// (cached prefix), only the page signals vary. Deps: DEEPSEEK_API_KEY. See db.js (hosts, pdf_quality).\nimport { readFileSync, existsSync } from 'fs';\n\nconst DEEPSEEK_URL = 'https://api.deepseek.com/chat/completions';\n\n// Stable system prompt — keep byte-identical across calls so DeepSeek caches it as the shared prefix.\nconst SYS = [\n  'You write a short, factual CONTEXT note about a document to help an OCR + metadata service process it accurately.',\n  'You are given text gathered from the library web page(s) that link to the document.',\n  'Output 2–5 plain-text sentences (no markdown, no JSON, no preamble) stating ONLY what the input supports:',\n  \"the document's title, author or editor, publication/date, source collection or journal, primary language,\",\n  'subject, and any proper nouns, names, or archaic/transliterated spellings likely to appear in the text.',\n  'Omit anything not supported by the input. Never invent. Keep under 200 words.',\n].join(' ');\n\nconst decodeEntities = (s) => s\n  .replace(/&amp;/g, '&').replace(/&#39;/g, \"'\").replace(/&quot;/g, '\"')\n  .replace(/&aacute;/g, 'á').replace(/&iacute;/g, 'í').replace(/&eacute;/g, 'é')\n  .replace(/&[a-z]+;/gi, ' ');\n\nconst metaTags = (html) => {\n  const out = {};\n  const t = html.match(/<title>([^<]*)<\\/title>/i); if (t) out.title = t[1].trim();\n  for (const m of html.matchAll(/<meta\\b[^>]*>/gi)) {\n    const key = (m[0].match(/(?:property|name)=[\"']([^\"']+)[\"']/i) || [])[1];\n    const val = (m[0].match(/content=[\"']([^\"']*)[\"']/i) || [])[1];\n    if (key && val) out[key.toLowerCase()] = val;\n  }\n  return out;\n};\n\nconst bodyText = (html) => decodeEntities(\n  html.replace(/<script[\\s\\S]*?<\\/script>/gi, ' ').replace(/<style[\\s\\S]*?<\\/style>/gi, ' ').replace(/<[^>]+>/g, ' ')\n).replace(/\\s+/g, ' ').trim();\n\n/** Gather raw signals about a document from its hosting page(s) + existing DB hints. Returns a string or null. */\nexport function gatherSignals(db, url) {\n  let out = `Filename: ${url.split('/').pop()}`;\n  const hosts = db.prepare('SELECT host_url FROM hosts WHERE hosted_url=?').all(url);\n  for (const h of hosts) {\n    const pg = db.prepare('SELECT local_path FROM pages WHERE url=?').get(h.host_url);\n    if (!pg?.local_path || !existsSync(pg.local_path)) continue;\n    const html = readFileSync(pg.local_path, 'utf8');\n    const mt = metaTags(html);\n    out += `\\nPAGE ${h.host_url}`;\n    if (mt.title) out += `\\n  title: ${mt.title}`;\n    if (mt['og:title']) out += `\\n  og:title: ${mt['og:title']}`;\n    const desc = mt.description || mt['og:description'];\n    if (desc) out += `\\n  description: ${desc}`;\n    out += `\\n  body: ${bodyText(html).slice(0, 1000)}`;\n  }\n  const q = db.prepare('SELECT pdf_title, excerpt, ai_author FROM pdf_quality WHERE url=?').get(url);\n  if (q?.excerpt) out += `\\nText excerpt: ${q.excerpt.slice(0, 400)}`;\n  return out.trim() ? out : null;\n}\n\n/**\n * Build the SLP `context` string for a document. Returns a string (≤ ~3500 chars) or null if no signals.\n * @param {object}   o\n * @param {Database} o.db       open per-site DB (read access to hosts, pages, pdf_quality)\n * @param {string}   o.url      document URL\n * @param {string}   o.apiKey   DEEPSEEK_API_KEY\n * @param {string}  [o.model]   DeepSeek model (default deepseek-v4-flash)\n */\nexport async function buildJobContext({ db, url, apiKey, model = 'deepseek-v4-flash' }) {\n  if (!apiKey) return null;\n  const signals = gatherSignals(db, url);\n  if (!signals) return null;\n  const res = await fetch(DEEPSEEK_URL, {\n    method: 'POST',\n    headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },\n    body: JSON.stringify({\n      // deepseek-v4-flash is a reasoning model; disable thinking so the token budget goes to output\n      // (otherwise reasoning_tokens consume max_tokens and content comes back empty/truncated).\n      model, temperature: 0, max_tokens: 400, thinking: { type: 'disabled' },\n      messages: [{ role: 'system', content: SYS }, { role: 'user', content: signals.slice(0, 8000) }],\n    }),\n  });\n  if (!res.ok) throw new Error(`DeepSeek context ${res.status}: ${(await res.text()).slice(0, 160)}`);\n  const j = await res.json();\n  const text = j.choices?.[0]?.message?.content?.trim() || '';\n  const usage = j.usage || {};\n  return { context: text.slice(0, 3800) || null, cacheHit: usage.prompt_cache_hit_tokens ?? 0, cacheMiss: usage.prompt_cache_miss_tokens ?? 0 };\n}\n"}}}}