1 line
40 KiB
JSON
1 line
40 KiB
JSON
{"repo": "ankitrraj/arnotebook", "files": {"apps/api/src/modules/chat/routes.ts": {"f4fb26c0685e53eeec5da0d2c5b6d8b572d6111d": {"t": 1785745598.7512167, "n": 7717, "h": "1547ddc92479ab28", "c": "import { Router, Response } from \"express\";\nimport { getDb, schema } from \"@arnotebook/database\";\nimport { eq, and, desc } from \"drizzle-orm\";\nimport { chatMessageSchema, createConversationSchema } from \"@arnotebook/shared\";\nimport { AppError } from \"../../middleware/errorHandler.js\";\nimport type { AuthenticatedRequest } from \"../../middleware/auth.js\";\nimport { searchChunks } from \"@arnotebook/vector-store\";\nimport { embedText } from \"@arnotebook/ai\";\nimport { env } from \"@arnotebook/config\";\nimport {\n SYSTEM_PROMPT,\n buildContextPrompt,\n} from \"@arnotebook/ai\";\n\nconst router = Router();\n\n// Detect greetings/small-talk to skip RAG pipeline (saves embedding API calls)\nconst CONVERSATIONAL_RE = /^(h(i|ello|ey|ola|owdy)|yo+|sup|hey+|thanks?(\\s+you)?|thank\\s*you|good\\s*(morning|afternoon|evening|night|day)|bye|goodbye|see\\s+ya|ok(ay)?|cool|great|nice|awesome|got\\s+it|understood|hmm*|haan|namaste|shukriya|kya\\s*hal)\\s*[!?.,]*$/i;\n\nfunction isConversational(message: string): boolean {\n const trimmed = message.trim();\n return trimmed.split(/\\s+/).length <= 8 && CONVERSATIONAL_RE.test(trimmed);\n}\n\nfunction nid(req: AuthenticatedRequest): string {\n return req.params.notebookId as string;\n}\n\nrouter.get(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const db = getDb();\n const conversations = await db.query.conversations.findMany({\n where: and(\n eq(schema.conversations.notebookId, nid(req)),\n eq(schema.conversations.userId, req.userId!)\n ),\n orderBy: [desc(schema.conversations.updatedAt)],\n });\n res.json({ conversations });\n});\n\nrouter.post(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const body = createConversationSchema.parse(req.body);\n const db = getDb();\n\n const [conversation] = await db\n .insert(schema.conversations)\n .values({\n notebookId: nid(req),\n userId: req.userId!,\n title: body.title || \"New conversation\",\n })\n .returning();\n\n res.status(201).json({ conversation });\n});\n\nrouter.post(\"/:notebookId/chat/stream\", async (req: AuthenticatedRequest, res: Response) => {\n const body = chatMessageSchema.parse(req.body);\n const db = getDb();\n const userId = req.userId!;\n const notebookId = nid(req);\n\n const notebook = await db.query.notebooks.findFirst({\n where: and(\n eq(schema.notebooks.id, notebookId),\n eq(schema.notebooks.ownerId, userId)\n ),\n });\n if (!notebook) throw new AppError(\"Notebook not found\", 404);\n\n let conversationId = body.conversationId;\n\n if (!conversationId) {\n const [conv] = await db\n .insert(schema.conversations)\n .values({\n notebookId,\n userId,\n title: body.message.slice(0, 100),\n })\n .returning();\n conversationId = conv.id;\n }\n\n await db.insert(schema.messages).values({\n conversationId,\n role: \"user\",\n content: body.message,\n });\n\n res.setHeader(\"Content-Type\", \"text/event-stream\");\n res.setHeader(\"Cache-Control\", \"no-cache\");\n res.setHeader(\"Connection\", \"keep-alive\");\n res.setHeader(\"X-Accel-Buffering\", \"no\");\n\n try {\n let retrievedChunks: any[] = [];\n const conversational = isConversational(body.message);\n\n // Skip embedding + vector search for greetings (saves API cost)\n if (!conversational) {\n const queryVector = await embedText(body.message);\n retrievedChunks = await searchChunks({\n notebookId,\n vector: queryVector,\n topK: 12,\n });\n }\n\n // Build LLM messages based on context availability\n let llmMessages: { role: string; content: string }[];\n\n if (retrievedChunks.length > 0) {\n // Full RAG — sources found\n const context = buildContextPrompt(retrievedChunks);\n llmMessages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: `${context}\\n\\nQuestion: ${body.message}` },\n ];\n } else if (conversational) {\n // Greeting/small-talk — no sources needed\n llmMessages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: body.message },\n ];\n } else {\n // Factual question but no sources matched\n llmMessages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: `[No relevant source excerpts were found in this notebook for the question below. Respond honestly and suggest adding relevant sources.]\\n\\nQuestion: ${body.message}` },\n ];\n }\n\n const deepseekResponse = await fetch(\n `${env.DEEPSEEK_BASE_URL}/v1/chat/completions`,\n {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${env.DEEPSEEK_API_KEY}`,\n },\n body: JSON.stringify({\n model: \"deepseek-v4-flash\",\n messages: llmMessages,\n stream: true,\n temperature: conversational ? 0.7 : 0.3,\n max_tokens: 8192,\n }),\n }\n );\n\n if (!deepseekResponse.ok) {\n throw new Error(`DeepSeek API error: ${deepseekResponse.status}`);\n }\n\n const reader = deepseekResponse.body!.getReader();\n const decoder = new TextDecoder();\n let fullResponse = \"\";\n const usedCitations = new Set<string>();\n\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n\n const chunk = decoder.decode(value);\n const lines = chunk.split(\"\\n\").filter((l) => l.startsWith(\"data: \"));\n\n for (const line of lines) {\n const data = line.slice(6);\n if (data === \"[DONE]\") continue;\n\n try {\n const parsed = JSON.parse(data);\n const token: string = parsed.choices?.[0]?.delta?.content || \"\";\n if (token) {\n fullResponse += token;\n if (!conversational) {\n const citationMatch = token.match(/\\[C(\\d+)\\]/g);\n if (citationMatch) {\n citationMatch.forEach((c: string) => usedCitations.add(c));\n }\n }\n res.write(`data: ${JSON.stringify({ type: \"token\", content: token })}\\n\\n`);\n }\n } catch {\n // skip parse errors\n }\n }\n }\n\n const [msg] = await db\n .insert(schema.messages)\n .values({ conversationId: conversationId!, role: \"assistant\", content: fullResponse })\n .returning();\n\n const savedCitations: any[] = [];\n for (const ref of usedCitations) {\n const idx = parseInt(ref.slice(2, -1)) - 1; // [C1] → \"1\" → index 0\n if (isNaN(idx) || idx < 0) continue;\n const chunk = retrievedChunks[idx];\n if (chunk) {\n const [citation] = await db\n .insert(schema.messageCitations)\n .values({\n messageId: msg.id,\n sourceId: chunk.sourceId,\n chunkIndex: chunk.chunkIndex,\n text: chunk.text.slice(0, 500),\n metadataJson: {\n title: chunk.title,\n pageNumber: chunk.pageNumber,\n section: chunk.section,\n url: chunk.url,\n startTime: chunk.startTime,\n endTime: chunk.endTime,\n } as any,\n })\n .returning();\n savedCitations.push(citation);\n }\n }\n\n res.write(`data: ${JSON.stringify({ type: \"citation\", content: \"\", citations: savedCitations })}\\n\\n`);\n res.write(`data: ${JSON.stringify({ type: \"done\", content: \"\", conversationId, messageId: msg.id, citations: savedCitations })}\\n\\n`);\n } catch (error: any) {\n console.error(\"Chat error:\", error);\n res.write(`data: ${JSON.stringify({ type: \"error\", content: error.message || \"Internal error\" })}\\n\\n`);\n } finally {\n res.end();\n }\n});\n\nexport { router as chatRouter };\n"}, "256eb595aa9e711b9e836c57671e5ca080dcec75": {"t": 1785754640.0903904, "n": 7717, "h": "1547ddc92479ab28", "c": "import { Router, Response } from \"express\";\nimport { getDb, schema } from \"@arnotebook/database\";\nimport { eq, and, desc } from \"drizzle-orm\";\nimport { chatMessageSchema, createConversationSchema } from \"@arnotebook/shared\";\nimport { AppError } from \"../../middleware/errorHandler.js\";\nimport type { AuthenticatedRequest } from \"../../middleware/auth.js\";\nimport { searchChunks } from \"@arnotebook/vector-store\";\nimport { embedText } from \"@arnotebook/ai\";\nimport { env } from \"@arnotebook/config\";\nimport {\n SYSTEM_PROMPT,\n buildContextPrompt,\n} from \"@arnotebook/ai\";\n\nconst router = Router();\n\n// Detect greetings/small-talk to skip RAG pipeline (saves embedding API calls)\nconst CONVERSATIONAL_RE = /^(h(i|ello|ey|ola|owdy)|yo+|sup|hey+|thanks?(\\s+you)?|thank\\s*you|good\\s*(morning|afternoon|evening|night|day)|bye|goodbye|see\\s+ya|ok(ay)?|cool|great|nice|awesome|got\\s+it|understood|hmm*|haan|namaste|shukriya|kya\\s*hal)\\s*[!?.,]*$/i;\n\nfunction isConversational(message: string): boolean {\n const trimmed = message.trim();\n return trimmed.split(/\\s+/).length <= 8 && CONVERSATIONAL_RE.test(trimmed);\n}\n\nfunction nid(req: AuthenticatedRequest): string {\n return req.params.notebookId as string;\n}\n\nrouter.get(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const db = getDb();\n const conversations = await db.query.conversations.findMany({\n where: and(\n eq(schema.conversations.notebookId, nid(req)),\n eq(schema.conversations.userId, req.userId!)\n ),\n orderBy: [desc(schema.conversations.updatedAt)],\n });\n res.json({ conversations });\n});\n\nrouter.post(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const body = createConversationSchema.parse(req.body);\n const db = getDb();\n\n const [conversation] = await db\n .insert(schema.conversations)\n .values({\n notebookId: nid(req),\n userId: req.userId!,\n title: body.title || \"New conversation\",\n })\n .returning();\n\n res.status(201).json({ conversation });\n});\n\nrouter.post(\"/:notebookId/chat/stream\", async (req: AuthenticatedRequest, res: Response) => {\n const body = chatMessageSchema.parse(req.body);\n const db = getDb();\n const userId = req.userId!;\n const notebookId = nid(req);\n\n const notebook = await db.query.notebooks.findFirst({\n where: and(\n eq(schema.notebooks.id, notebookId),\n eq(schema.notebooks.ownerId, userId)\n ),\n });\n if (!notebook) throw new AppError(\"Notebook not found\", 404);\n\n let conversationId = body.conversationId;\n\n if (!conversationId) {\n const [conv] = await db\n .insert(schema.conversations)\n .values({\n notebookId,\n userId,\n title: body.message.slice(0, 100),\n })\n .returning();\n conversationId = conv.id;\n }\n\n await db.insert(schema.messages).values({\n conversationId,\n role: \"user\",\n content: body.message,\n });\n\n res.setHeader(\"Content-Type\", \"text/event-stream\");\n res.setHeader(\"Cache-Control\", \"no-cache\");\n res.setHeader(\"Connection\", \"keep-alive\");\n res.setHeader(\"X-Accel-Buffering\", \"no\");\n\n try {\n let retrievedChunks: any[] = [];\n const conversational = isConversational(body.message);\n\n // Skip embedding + vector search for greetings (saves API cost)\n if (!conversational) {\n const queryVector = await embedText(body.message);\n retrievedChunks = await searchChunks({\n notebookId,\n vector: queryVector,\n topK: 12,\n });\n }\n\n // Build LLM messages based on context availability\n let llmMessages: { role: string; content: string }[];\n\n if (retrievedChunks.length > 0) {\n // Full RAG — sources found\n const context = buildContextPrompt(retrievedChunks);\n llmMessages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: `${context}\\n\\nQuestion: ${body.message}` },\n ];\n } else if (conversational) {\n // Greeting/small-talk — no sources needed\n llmMessages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: body.message },\n ];\n } else {\n // Factual question but no sources matched\n llmMessages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: `[No relevant source excerpts were found in this notebook for the question below. Respond honestly and suggest adding relevant sources.]\\n\\nQuestion: ${body.message}` },\n ];\n }\n\n const deepseekResponse = await fetch(\n `${env.DEEPSEEK_BASE_URL}/v1/chat/completions`,\n {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${env.DEEPSEEK_API_KEY}`,\n },\n body: JSON.stringify({\n model: \"deepseek-v4-flash\",\n messages: llmMessages,\n stream: true,\n temperature: conversational ? 0.7 : 0.3,\n max_tokens: 8192,\n }),\n }\n );\n\n if (!deepseekResponse.ok) {\n throw new Error(`DeepSeek API error: ${deepseekResponse.status}`);\n }\n\n const reader = deepseekResponse.body!.getReader();\n const decoder = new TextDecoder();\n let fullResponse = \"\";\n const usedCitations = new Set<string>();\n\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n\n const chunk = decoder.decode(value);\n const lines = chunk.split(\"\\n\").filter((l) => l.startsWith(\"data: \"));\n\n for (const line of lines) {\n const data = line.slice(6);\n if (data === \"[DONE]\") continue;\n\n try {\n const parsed = JSON.parse(data);\n const token: string = parsed.choices?.[0]?.delta?.content || \"\";\n if (token) {\n fullResponse += token;\n if (!conversational) {\n const citationMatch = token.match(/\\[C(\\d+)\\]/g);\n if (citationMatch) {\n citationMatch.forEach((c: string) => usedCitations.add(c));\n }\n }\n res.write(`data: ${JSON.stringify({ type: \"token\", content: token })}\\n\\n`);\n }\n } catch {\n // skip parse errors\n }\n }\n }\n\n const [msg] = await db\n .insert(schema.messages)\n .values({ conversationId: conversationId!, role: \"assistant\", content: fullResponse })\n .returning();\n\n const savedCitations: any[] = [];\n for (const ref of usedCitations) {\n const idx = parseInt(ref.slice(2, -1)) - 1; // [C1] → \"1\" → index 0\n if (isNaN(idx) || idx < 0) continue;\n const chunk = retrievedChunks[idx];\n if (chunk) {\n const [citation] = await db\n .insert(schema.messageCitations)\n .values({\n messageId: msg.id,\n sourceId: chunk.sourceId,\n chunkIndex: chunk.chunkIndex,\n text: chunk.text.slice(0, 500),\n metadataJson: {\n title: chunk.title,\n pageNumber: chunk.pageNumber,\n section: chunk.section,\n url: chunk.url,\n startTime: chunk.startTime,\n endTime: chunk.endTime,\n } as any,\n })\n .returning();\n savedCitations.push(citation);\n }\n }\n\n res.write(`data: ${JSON.stringify({ type: \"citation\", content: \"\", citations: savedCitations })}\\n\\n`);\n res.write(`data: ${JSON.stringify({ type: \"done\", content: \"\", conversationId, messageId: msg.id, citations: savedCitations })}\\n\\n`);\n } catch (error: any) {\n console.error(\"Chat error:\", error);\n res.write(`data: ${JSON.stringify({ type: \"error\", content: error.message || \"Internal error\" })}\\n\\n`);\n } finally {\n res.end();\n }\n});\n\nexport { router as chatRouter };\n"}, "2695ca4d38a52d1008af0d038a34730feba671de": {"t": 1785754641.5727155, "n": 6574, "h": "8b3e3cd89145412f", "c": "import { Router, Response } from \"express\";\nimport { getDb, schema } from \"@arnotebook/database\";\nimport { eq, and, desc } from \"drizzle-orm\";\nimport { chatMessageSchema, createConversationSchema } from \"@arnotebook/shared\";\nimport { AppError } from \"../../middleware/errorHandler.js\";\nimport type { AuthenticatedRequest } from \"../../middleware/auth.js\";\nimport { searchChunks } from \"@arnotebook/vector-store\";\nimport { embedText } from \"@arnotebook/ai\";\nimport { env } from \"@arnotebook/config\";\nimport {\n SYSTEM_PROMPT,\n buildContextPrompt,\n} from \"@arnotebook/ai\";\n\nconst router = Router();\n\nfunction nid(req: AuthenticatedRequest): string {\n return req.params.notebookId as string;\n}\n\nrouter.get(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const db = getDb();\n const conversations = await db.query.conversations.findMany({\n where: and(\n eq(schema.conversations.notebookId, nid(req)),\n eq(schema.conversations.userId, req.userId!)\n ),\n orderBy: [desc(schema.conversations.updatedAt)],\n });\n res.json({ conversations });\n});\n\nrouter.post(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const body = createConversationSchema.parse(req.body);\n const db = getDb();\n\n const [conversation] = await db\n .insert(schema.conversations)\n .values({\n notebookId: nid(req),\n userId: req.userId!,\n title: body.title || \"New conversation\",\n })\n .returning();\n\n res.status(201).json({ conversation });\n});\n\nrouter.post(\"/:notebookId/chat/stream\", async (req: AuthenticatedRequest, res: Response) => {\n const body = chatMessageSchema.parse(req.body);\n const db = getDb();\n const userId = req.userId!;\n const notebookId = nid(req);\n\n const notebook = await db.query.notebooks.findFirst({\n where: and(\n eq(schema.notebooks.id, notebookId),\n eq(schema.notebooks.ownerId, userId)\n ),\n });\n if (!notebook) throw new AppError(\"Notebook not found\", 404);\n\n let conversationId = body.conversationId;\n\n if (!conversationId) {\n const [conv] = await db\n .insert(schema.conversations)\n .values({\n notebookId,\n userId,\n title: body.message.slice(0, 100),\n })\n .returning();\n conversationId = conv.id;\n }\n\n await db.insert(schema.messages).values({\n conversationId,\n role: \"user\",\n content: body.message,\n });\n\n res.setHeader(\"Content-Type\", \"text/event-stream\");\n res.setHeader(\"Cache-Control\", \"no-cache\");\n res.setHeader(\"Connection\", \"keep-alive\");\n res.setHeader(\"X-Accel-Buffering\", \"no\");\n\n try {\n const queryVector = await embedText(body.message);\n const retrievedChunks = await searchChunks({\n notebookId,\n vector: queryVector,\n topK: 12,\n });\n\n if (retrievedChunks.length === 0) {\n const noResult = \"I couldn't find that in this notebook's sources.\";\n const [msg] = await db\n .insert(schema.messages)\n .values({ conversationId: conversationId!, role: \"assistant\", content: noResult })\n .returning();\n\n res.write(`data: ${JSON.stringify({ type: \"token\", content: noResult })}\\n\\n`);\n res.write(`data: ${JSON.stringify({ type: \"done\", content: \"\", conversationId, messageId: msg.id })}\\n\\n`);\n res.end();\n return;\n }\n\n const context = buildContextPrompt(retrievedChunks);\n const messages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: `${context}\\n\\nQuestion: ${body.message}` },\n ];\n\n const deepseekResponse = await fetch(\n `${env.DEEPSEEK_BASE_URL}/v1/chat/completions`,\n {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${env.DEEPSEEK_API_KEY}`,\n },\n body: JSON.stringify({\n model: \"deepseek-v4-flash\",\n messages,\n stream: true,\n max_tokens: 8192,\n }),\n }\n );\n\n if (!deepseekResponse.ok) {\n throw new Error(`DeepSeek API error: ${deepseekResponse.status}`);\n }\n\n const reader = deepseekResponse.body!.getReader();\n const decoder = new TextDecoder();\n let fullResponse = \"\";\n const usedCitations = new Set<string>();\n\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n\n const chunk = decoder.decode(value);\n const lines = chunk.split(\"\\n\").filter((l) => l.startsWith(\"data: \"));\n\n for (const line of lines) {\n const data = line.slice(6);\n if (data === \"[DONE]\") continue;\n\n try {\n const parsed = JSON.parse(data);\n const token: string = parsed.choices?.[0]?.delta?.content || \"\";\n if (token) {\n fullResponse += token;\n const citationMatch = token.match(/\\[C(\\d+)\\]/g);\n if (citationMatch) {\n citationMatch.forEach((c: string) => usedCitations.add(c));\n }\n res.write(`data: ${JSON.stringify({ type: \"token\", content: token })}\\n\\n`);\n }\n } catch {\n // skip parse errors\n }\n }\n }\n\n const [msg] = await db\n .insert(schema.messages)\n .values({ conversationId: conversationId!, role: \"assistant\", content: fullResponse })\n .returning();\n\n const savedCitations: any[] = [];\n for (const ref of usedCitations) {\n const idx = parseInt(ref.slice(1)) - 1;\n const chunk = retrievedChunks[idx];\n if (chunk) {\n const [citation] = await db\n .insert(schema.messageCitations)\n .values({\n messageId: msg.id,\n sourceId: chunk.sourceId,\n chunkIndex: chunk.chunkIndex,\n text: chunk.text.slice(0, 500),\n metadataJson: {\n title: chunk.title,\n pageNumber: chunk.pageNumber,\n section: chunk.section,\n url: chunk.url,\n startTime: chunk.startTime,\n endTime: chunk.endTime,\n } as any,\n })\n .returning();\n savedCitations.push(citation);\n }\n }\n\n res.write(`data: ${JSON.stringify({ type: \"citation\", content: \"\", citations: savedCitations })}\\n\\n`);\n res.write(`data: ${JSON.stringify({ type: \"done\", content: \"\", conversationId, messageId: msg.id, citations: savedCitations })}\\n\\n`);\n } catch (error: any) {\n console.error(\"Chat error:\", error);\n res.write(`data: ${JSON.stringify({ type: \"error\", content: error.message || \"Internal error\" })}\\n\\n`);\n } finally {\n res.end();\n }\n});\n\nexport { router as chatRouter };\n"}, "c2d65affc92d8e3607b3a60b5913f9a484e40a55": {"t": 1785754643.7676535, "n": 6598, "h": "1ba1386b494d67ba", "c": "import { Router, Response } from \"express\";\nimport { getDb, schema } from \"@arnotebook/database\";\nimport { eq, and, desc } from \"drizzle-orm\";\nimport { chatMessageSchema, createConversationSchema } from \"@arnotebook/shared\";\nimport { AppError } from \"../../middleware/errorHandler.js\";\nimport type { AuthenticatedRequest } from \"../../middleware/auth.js\";\nimport { searchChunks } from \"@arnotebook/vector-store\";\nimport { embedText } from \"@arnotebook/ai\";\nimport { env } from \"@arnotebook/config\";\nimport {\n SYSTEM_PROMPT,\n buildContextPrompt,\n} from \"@arnotebook/ai\";\n\nconst router = Router();\n\nfunction nid(req: AuthenticatedRequest): string {\n return req.params.notebookId as string;\n}\n\nrouter.get(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const db = getDb();\n const conversations = await db.query.conversations.findMany({\n where: and(\n eq(schema.conversations.notebookId, nid(req)),\n eq(schema.conversations.userId, req.userId!)\n ),\n orderBy: [desc(schema.conversations.updatedAt)],\n });\n res.json({ conversations });\n});\n\nrouter.post(\"/:notebookId/conversations\", async (req: AuthenticatedRequest, res: Response) => {\n const body = createConversationSchema.parse(req.body);\n const db = getDb();\n\n const [conversation] = await db\n .insert(schema.conversations)\n .values({\n notebookId: nid(req),\n userId: req.userId!,\n title: body.title || \"New conversation\",\n })\n .returning();\n\n res.status(201).json({ conversation });\n});\n\nrouter.post(\"/:notebookId/chat/stream\", async (req: AuthenticatedRequest, res: Response) => {\n const body = chatMessageSchema.parse(req.body);\n const db = getDb();\n const userId = req.userId!;\n const notebookId = nid(req);\n\n const notebook = await db.query.notebooks.findFirst({\n where: and(\n eq(schema.notebooks.id, notebookId),\n eq(schema.notebooks.ownerId, userId)\n ),\n });\n if (!notebook) throw new AppError(\"Notebook not found\", 404);\n\n let conversationId = body.conversationId;\n\n if (!conversationId) {\n const [conv] = await db\n .insert(schema.conversations)\n .values({\n notebookId,\n userId,\n title: body.message.slice(0, 100),\n })\n .returning();\n conversationId = conv.id;\n }\n\n await db.insert(schema.messages).values({\n conversationId,\n role: \"user\",\n content: body.message,\n });\n\n res.setHeader(\"Content-Type\", \"text/event-stream\");\n res.setHeader(\"Cache-Control\", \"no-cache\");\n res.setHeader(\"Connection\", \"keep-alive\");\n res.setHeader(\"X-Accel-Buffering\", \"no\");\n\n try {\n const queryVector = await embedText(body.message);\n const retrievedChunks = await searchChunks({\n notebookId,\n vector: queryVector,\n topK: 12,\n });\n\n if (retrievedChunks.length === 0) {\n const noResult = \"I couldn't find that in this notebook's sources.\";\n const [msg] = await db\n .insert(schema.messages)\n .values({ conversationId: conversationId!, role: \"assistant\", content: noResult })\n .returning();\n\n res.write(`data: ${JSON.stringify({ type: \"token\", content: noResult })}\\n\\n`);\n res.write(`data: ${JSON.stringify({ type: \"done\", content: \"\", conversationId, messageId: msg.id })}\\n\\n`);\n res.end();\n return;\n }\n\n const context = buildContextPrompt(retrievedChunks);\n const messages = [\n { role: \"system\", content: SYSTEM_PROMPT },\n { role: \"user\", content: `${context}\\n\\nQuestion: ${body.message}` },\n ];\n\n const deepseekResponse = await fetch(\n `${env.DEEPSEEK_BASE_URL}/v1/chat/completions`,\n {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${env.DEEPSEEK_API_KEY}`,\n },\n body: JSON.stringify({\n model: \"deepseek-chat\",\n messages,\n stream: true,\n temperature: 0.3,\n max_tokens: 2048,\n }),\n }\n );\n\n if (!deepseekResponse.ok) {\n throw new Error(`DeepSeek API error: ${deepseekResponse.status}`);\n }\n\n const reader = deepseekResponse.body!.getReader();\n const decoder = new TextDecoder();\n let fullResponse = \"\";\n const usedCitations = new Set<string>();\n\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n\n const chunk = decoder.decode(value);\n const lines = chunk.split(\"\\n\").filter((l) => l.startsWith(\"data: \"));\n\n for (const line of lines) {\n const data = line.slice(6);\n if (data === \"[DONE]\") continue;\n\n try {\n const parsed = JSON.parse(data);\n const token: string = parsed.choices?.[0]?.delta?.content || \"\";\n if (token) {\n fullResponse += token;\n const citationMatch = token.match(/\\[C(\\d+)\\]/g);\n if (citationMatch) {\n citationMatch.forEach((c: string) => usedCitations.add(c));\n }\n res.write(`data: ${JSON.stringify({ type: \"token\", content: token })}\\n\\n`);\n }\n } catch {\n // skip parse errors\n }\n }\n }\n\n const [msg] = await db\n .insert(schema.messages)\n .values({ conversationId: conversationId!, role: \"assistant\", content: fullResponse })\n .returning();\n\n const savedCitations: any[] = [];\n for (const ref of usedCitations) {\n const idx = parseInt(ref.slice(1)) - 1;\n const chunk = retrievedChunks[idx];\n if (chunk) {\n const [citation] = await db\n .insert(schema.messageCitations)\n .values({\n messageId: msg.id,\n sourceId: chunk.sourceId,\n chunkIndex: chunk.chunkIndex,\n text: chunk.text.slice(0, 500),\n metadataJson: {\n title: chunk.title,\n pageNumber: chunk.pageNumber,\n section: chunk.section,\n url: chunk.url,\n startTime: chunk.startTime,\n endTime: chunk.endTime,\n } as any,\n })\n .returning();\n savedCitations.push(citation);\n }\n }\n\n res.write(`data: ${JSON.stringify({ type: \"citation\", content: \"\", citations: savedCitations })}\\n\\n`);\n res.write(`data: ${JSON.stringify({ type: \"done\", content: \"\", conversationId, messageId: msg.id, citations: savedCitations })}\\n\\n`);\n } catch (error: any) {\n console.error(\"Chat error:\", error);\n res.write(`data: ${JSON.stringify({ type: \"error\", content: error.message || \"Internal error\" })}\\n\\n`);\n } finally {\n res.end();\n }\n});\n\nexport { router as chatRouter };\n"}}, "README.md": {"f4fb26c0685e53eeec5da0d2c5b6d8b572d6111d": {"t": 1785747274.146768, "n": 5045, "h": "4de0c42fba442c88", "c": "# ChaibookLM — AI Research Notebook\n\n> **Live link kal 12pm 27 july baje tak** — currently running locally at `http://localhost:3000`\n\nAI-powered research notebook with RAG pipeline, notebook-isolated sources, streaming DeepSeek chat with citations, and PostHog-inspired UI.\n\n## Tech Stack\n\n- **Frontend:** Next.js 15, TypeScript, Tailwind CSS (PostHog design)\n- **Backend:** Express + TypeScript\n- **Database:** Neon PostgreSQL + Drizzle ORM\n- **Vector DB:** Qdrant (notebook-isolated collections)\n- **Queue:** BullMQ + Redis\n- **LLM:** DeepSeek V4 Flash (streaming, thinking mode)\n- **Embeddings:** Voyage AI (voyage-4-lite, 1024 dim)\n\n## Quick Start (Local)\n\n### Prerequisites\n\n- Node.js >= 20, pnpm >= 9, Docker\n\n```bash\n# 1. Install\npnpm install\n\n# 2. Start Docker services (Qdrant + Redis)\ndocker compose -f docker/docker-compose.yml up -d\n\n# 3. Set up env\ncp .env.example .env\n# Add: DEEPSEEK_API_KEY, EMBEDDING_API_KEY, DATABASE_URL\n\n# 4. Push DB schema\npnpm db:push\n\n# 5. Run (3 terminals)\npnpm dev:api # Express API :4000\npnpm dev:worker # BullMQ worker\npnpm dev:web # Next.js :3000\n```\n\nOpen http://localhost:3000 — login with any email.\n\n## Production (DigitalOcean Droplet)\n\n### Architecture\n\n```\nVercel (optional) Droplet\n┌─────────┐ ┌──────────────────┐\n│ Frontend │ ──→ │ API :4000 │\n│ (Next.js)│ HTTP │ Worker (BullMQ) │\n└─────────┘ │ Qdrant (Docker) │\n │ Redis (Docker) │\n │ Nginx (reverse │\n │ proxy + SSL) │\n └──────────────────┘\n```\n\n### Droplet Setup\n\n```bash\n# System\napt update && apt upgrade -y\napt install -y docker.io docker-compose-v2 nginx certbot python3-certbot-nginx\ncurl -fsSL https://deb.nodesource.com/setup_22.x | bash -\napt install -y nodejs\nnpm install -g pnpm pm2\n\n# Clone\ngit clone <repo-url> /root/arnotebook\ncd /root/arnotebook\npnpm install\n\n# Docker services\ndocker compose -f docker/docker-compose.yml up -d\n\n# Build & Start\npnpm build\npm2 start apps/api/ecosystem.config.cjs\npm2 start apps/worker/ecosystem.config.cjs\n\n# Frontend\ncd apps/web && pnpm build\npm2 start npm --name \"web\" -- start\n\n# Nginx (with domain)\nnano /etc/nginx/sites-available/arnotebook\n# → proxy :3000 for web, :4000/api for API\ncertbot --nginx -d yourdomain.com\n```\n\n### Or run directly with tsx (no build needed)\n\n```bash\npm2 start \"npx tsx apps/api/src/server.ts\" --name api\npm2 start \"npx tsx apps/worker/src/worker.ts\" --name worker\n```\n\n## RAG Pipeline\n\n1. Source uploaded → queued\n2. Worker extracts content (text/URL/YouTube/VTT)\n3. Content chunked (400-800 tokens, 120 overlap)\n4. Each chunk embedded via Voyage AI (1024-dim vectors)\n5. Vectors upserted to Qdrant with `notebookId` filter\n6. User question → embedded → Qdrant search (notebook-scoped)\n7. Top chunks + citation IDs → DeepSeek V4 Flash\n8. Response streamed to UI (SSE), citations saved\n9. Click citation → source panel opens at exact location\n\n## Project Structure\n\n```\narnotebook/\n├── apps/\n│ ├── web/ # Next.js frontend\n│ ├── api/ # Express API (port 4000)\n│ └── worker/ # BullMQ ingestion worker\n├── packages/\n│ ├── database/ # Drizzle schema + Neon client\n│ ├── shared/ # Types, Zod schemas, constants\n│ ├── ai/ # Embeddings, prompts, RAG logic\n│ ├── ingestion/ # Chunking, content extraction\n│ ├── vector-store/ # Qdrant client + operations\n│ └── config/ # Environment validation\n├── docker/ # Docker Compose (Qdrant + Redis)\n└── README.md\n```\n\n## API Endpoints\n\n| Method | Path | Description |\n|--------|------|-------------|\n| POST | `/auth/login` | Login/register with email |\n| GET | `/auth/me` | Get current user |\n| GET | `/notebooks` | List notebooks |\n| POST | `/notebooks` | Create notebook |\n| GET | `/notebooks/:id` | Get notebook |\n| PATCH | `/notebooks/:id` | Update notebook |\n| DELETE | `/notebooks/:id` | Delete notebook |\n| GET | `/notebooks/:id/sources` | List sources |\n| POST | `/notebooks/:id/sources/text` | Add text source |\n| POST | `/notebooks/:id/sources/url` | Add URL source |\n| POST | `/:id/sources/:sid/reindex` | Re-index source |\n| DELETE | `/:id/sources/:sid` | Delete source |\n| GET | `/notebooks/:id/conversations` | List conversations |\n| POST | `/notebooks/:id/conversations` | Create conversation |\n| POST | `/notebooks/:id/chat/stream` | Stream chat response (SSE) |\n\n## Environment Variables\n\n| Variable | Description | Default |\n|----------|-------------|---------|\n| `DATABASE_URL` | Neon PostgreSQL | — |\n| `QDRANT_URL` | Qdrant server | http://localhost:6333 |\n| `REDIS_URL` | Redis | redis://localhost:6379 |\n| `DEEPSEEK_API_KEY` | DeepSeek API key | — |\n| `DEEPSEEK_BASE_URL` | DeepSeek API base | https://api.deepseek.com |\n| `EMBEDDING_API_KEY` | Voyage/embedding API key | — |\n| `EMBEDDING_BASE_URL` | Embedding API base | https://api.voyageai.com/v1 |\n| `EMBEDDING_MODEL` | Embedding model | voyage-4-lite |\n| `NEXT_PUBLIC_API_URL` | API URL for frontend | http://localhost:4000 |\n"}, "c2d65affc92d8e3607b3a60b5913f9a484e40a55": {"t": 1785754639.2482474, "n": 4323, "h": "55620fbbe56eb47e", "c": "# ChaibookLM — AI Research Notebook\n\nAn AI-powered research assistant inspired by NotebookLM. Upload multiple knowledge sources, ask questions grounded in those sources, and receive answers with proper citations.\n\n## Architecture\n\n```\n ┌─────────────┐\n │ Next.js UI │ Port 3000\n └──────┬──────┘\n │ HTTP/SSE\n ┌──────▼──────┐\n │ Express API │ Port 4000\n └──────┬──────┘\n │\n ┌────────────────────┼────────────────────┐\n ▼ ▼ ▼\n ┌──────────┐ ┌──────────┐ ┌──────────┐\n │ Neon │ │ Qdrant │ │ Redis │\n │PostgreSQL│ │ Vectors │ │ + │\n │ (Drizzle)│ │ │ │ BullMQ │\n └──────────┘ └──────────┘ └──────────┘\n```\n\n## Tech Stack\n\n- **Frontend:** Next.js 15, TypeScript, Tailwind CSS\n- **Backend:** Express + TypeScript\n- **Database:** Neon PostgreSQL + Drizzle ORM\n- **Vector DB:** Qdrant (notebook-isolated collections)\n- **Queue:** BullMQ + Redis\n- **LLM:** DeepSeek Chat (streaming)\n- **Embeddings:** OpenAI text-embedding-3-large\n\n## Quick Start\n\n### Prerequisites\n\n- Node.js >= 20\n- pnpm >= 9\n- Docker\n\n### 1. Clone and Install\n\n```bash\npnpm install\n```\n\n### 2. Start Docker Services\n\n```bash\ndocker compose -f docker/docker-compose.yml up -d\n```\n\n### 3. Set Up Environment\n\n```bash\ncp .env.example .env\n# Edit .env with your API keys\n```\n\n### 4. Database\n\n```bash\npnpm db:generate\npnpm db:push\n```\n\n### 5. Run\n\n```bash\n# Terminal 1: API server\npnpm dev:api\n\n# Terminal 2: Worker\npnpm dev:worker\n\n# Terminal 3: Frontend\npnpm dev:web\n```\n\nOpen http://localhost:3000 and enter any email to log in.\n\n## Project Structure\n\n```\narnotebook/\n├── apps/\n│ ├── web/ # Next.js frontend\n│ ├── api/ # Express API\n│ └── worker/ # Background ingestion worker\n├── packages/\n│ ├── database/ # Drizzle schema + Neon client\n│ ├── shared/ # Types, Zod schemas, constants\n│ ├── ai/ # Prompts, embeddings, RAG logic\n│ ├── ingestion/ # Chunking, content extraction\n│ ├── vector-store/ # Qdrant client + operations\n│ └── config/ # Environment validation\n├── docker/ # Docker Compose for local services\n└── README.md\n```\n\n## API Endpoints\n\n### Auth\n- `POST /auth/login` — Login/register with email\n- `GET /auth/me` — Get current user\n\n### Notebooks\n- `GET /notebooks` — List notebooks\n- `POST /notebooks` — Create notebook\n- `GET /notebooks/:id` — Get notebook\n- `PATCH /notebooks/:id` — Update notebook\n- `DELETE /notebooks/:id` — Delete notebook\n\n### Sources\n- `GET /notebooks/:id/sources` — List sources\n- `POST /notebooks/:id/sources/text` — Add text source\n- `POST /notebooks/:id/sources/url` — Add URL source\n- `POST /notebooks/:id/sources/:sourceId/reindex` — Re-index source\n- `DELETE /notebooks/:id/sources/:sourceId` — Delete source\n- `GET /sources/:sourceId/content` — Get source content\n\n### Chat\n- `GET /notebooks/:id/conversations` — List conversations\n- `POST /notebooks/:id/conversations` — Create conversation\n- `POST /notebooks/:id/chat/stream` — Stream chat response (SSE)\n\n### Citations\n- `GET /citations/:citationId` — Get citation details\n\n## RAG Pipeline\n\n1. Source is uploaded → queued\n2. Worker extracts content (text/website/YouTube/VTT)\n3. Content is chunked (semantic paragraphs, 400-800 tokens)\n4. Each chunk is embedded via text-embedding-3-large\n5. Vectors upserted to Qdrant with `notebookId` payload filter\n6. User asks question → embedded → Qdrant search filtered by notebook\n7. Top chunks sent to DeepSeek with citation IDs\n8. Response streamed to UI, citations saved to Postgres\n9. User clicks citation → source panel opens at exact location\n\n## Environment Variables\n\n| Variable | Description | Required |\n|----------|-------------|----------|\n| `DATABASE_URL` | Neon PostgreSQL connection string | Yes |\n| `QDRANT_URL` | Qdrant server URL | Yes |\n| `REDIS_URL` | Redis connection string | Yes |\n| `DEEPSEEK_API_KEY` | DeepSeek API key | Yes |\n| `EMBEDDING_API_KEY` | OpenAI/embedding API key | Yes |\n| `EMBEDDING_MODEL` | Embedding model name | No |\n\n## License\n\nMIT\n"}}}} |