OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
5.0 KiB
JSON
1 line
5.0 KiB
JSON
{"repo": "chadananda/site2rag", "files": {"src/slp-context.js": {"4e012e5efbf018ec6e17a12e31d1cb3e1ae898d7": {"t": 1786068266.847429, "n": 4794, "h": "67c249fe1abb0488", "c": "// Builds the free-text `context` block sent with an SLP job. Distills the document's hosting\n// page(s) — title, link text, description, author, language, subject, likely proper nouns — via a\n// cheap DeepSeek call. SLP feeds this into OCR synthesis + metadata extraction (better proper-noun\n// and archaic-spelling resolution). Cache-optimized: the system prompt is byte-stable across calls\n// (cached prefix), only the page signals vary. Deps: DEEPSEEK_API_KEY. See db.js (hosts, pdf_quality).\nimport { readFileSync, existsSync } from 'fs';\n\nconst DEEPSEEK_URL = 'https://api.deepseek.com/chat/completions';\n\n// Stable system prompt — keep byte-identical across calls so DeepSeek caches it as the shared prefix.\nconst SYS = [\n 'You write a short, factual CONTEXT note about a document to help an OCR + metadata service process it accurately.',\n 'You are given text gathered from the library web page(s) that link to the document.',\n 'Output 2–5 plain-text sentences (no markdown, no JSON, no preamble) stating ONLY what the input supports:',\n \"the document's title, author or editor, publication/date, source collection or journal, primary language,\",\n 'subject, and any proper nouns, names, or archaic/transliterated spellings likely to appear in the text.',\n 'Omit anything not supported by the input. Never invent. Keep under 200 words.',\n].join(' ');\n\nconst decodeEntities = (s) => s\n .replace(/&/g, '&').replace(/'/g, \"'\").replace(/"/g, '\"')\n .replace(/á/g, 'á').replace(/í/g, 'í').replace(/é/g, 'é')\n .replace(/&[a-z]+;/gi, ' ');\n\nconst metaTags = (html) => {\n const out = {};\n const t = html.match(/<title>([^<]*)<\\/title>/i); if (t) out.title = t[1].trim();\n for (const m of html.matchAll(/<meta\\b[^>]*>/gi)) {\n const key = (m[0].match(/(?:property|name)=[\"']([^\"']+)[\"']/i) || [])[1];\n const val = (m[0].match(/content=[\"']([^\"']*)[\"']/i) || [])[1];\n if (key && val) out[key.toLowerCase()] = val;\n }\n return out;\n};\n\nconst bodyText = (html) => decodeEntities(\n html.replace(/<script[\\s\\S]*?<\\/script>/gi, ' ').replace(/<style[\\s\\S]*?<\\/style>/gi, ' ').replace(/<[^>]+>/g, ' ')\n).replace(/\\s+/g, ' ').trim();\n\n/** Gather raw signals about a document from its hosting page(s) + existing DB hints. Returns a string or null. */\nexport function gatherSignals(db, url) {\n let out = `Filename: ${url.split('/').pop()}`;\n const hosts = db.prepare('SELECT host_url FROM hosts WHERE hosted_url=?').all(url);\n for (const h of hosts) {\n const pg = db.prepare('SELECT local_path FROM pages WHERE url=?').get(h.host_url);\n if (!pg?.local_path || !existsSync(pg.local_path)) continue;\n const html = readFileSync(pg.local_path, 'utf8');\n const mt = metaTags(html);\n out += `\\nPAGE ${h.host_url}`;\n if (mt.title) out += `\\n title: ${mt.title}`;\n if (mt['og:title']) out += `\\n og:title: ${mt['og:title']}`;\n const desc = mt.description || mt['og:description'];\n if (desc) out += `\\n description: ${desc}`;\n out += `\\n body: ${bodyText(html).slice(0, 1000)}`;\n }\n const q = db.prepare('SELECT pdf_title, excerpt, ai_author FROM pdf_quality WHERE url=?').get(url);\n if (q?.excerpt) out += `\\nText excerpt: ${q.excerpt.slice(0, 400)}`;\n return out.trim() ? out : null;\n}\n\n/**\n * Build the SLP `context` string for a document. Returns a string (≤ ~3500 chars) or null if no signals.\n * @param {object} o\n * @param {Database} o.db open per-site DB (read access to hosts, pages, pdf_quality)\n * @param {string} o.url document URL\n * @param {string} o.apiKey DEEPSEEK_API_KEY\n * @param {string} [o.model] DeepSeek model (default deepseek-v4-flash)\n */\nexport async function buildJobContext({ db, url, apiKey, model = 'deepseek-v4-flash' }) {\n if (!apiKey) return null;\n const signals = gatherSignals(db, url);\n if (!signals) return null;\n const res = await fetch(DEEPSEEK_URL, {\n method: 'POST',\n headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },\n body: JSON.stringify({\n // deepseek-v4-flash is a reasoning model; disable thinking so the token budget goes to output\n // (otherwise reasoning_tokens consume max_tokens and content comes back empty/truncated).\n model, temperature: 0, max_tokens: 400, thinking: { type: 'disabled' },\n messages: [{ role: 'system', content: SYS }, { role: 'user', content: signals.slice(0, 8000) }],\n }),\n });\n if (!res.ok) throw new Error(`DeepSeek context ${res.status}: ${(await res.text()).slice(0, 160)}`);\n const j = await res.json();\n const text = j.choices?.[0]?.message?.content?.trim() || '';\n const usage = j.usage || {};\n return { context: text.slice(0, 3800) || null, cacheHit: usage.prompt_cache_hit_tokens ?? 0, cacheMiss: usage.prompt_cache_miss_tokens ?? 0 };\n}\n"}}}} |