{"repo": "andikachandra23/papier", "files": {"backend/app/routers/summarize.py": {"4f6f1d2222f62936e7f4f12768ae31a25c547983": {"t": 1785595267.245979, "n": 23741, "h": "375b902ca8b624ae", "c": "\"\"\"PDF text extraction and AI summarization.\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nimport re\nimport tempfile\nfrom datetime import datetime, timezone\nfrom typing import Optional\n\nimport httpx\nimport pdfplumber\nfrom fastapi import APIRouter, Depends, HTTPException\nfrom pydantic import BaseModel\nfrom sqlalchemy.orm import Session\n\nfrom ..auth import get_current_user\nfrom ..config import env\nfrom ..database import get_db\nfrom ..models import Paper, User, AIUsageLog, Setting\nfrom ..storage import is_r2_path, stream_r2_object\nfrom .doi import BROWSER_USER_AGENT\n\nAI_DEFAULT_MODEL = env(\"AI_DEFAULT_MODEL\", \"mimo-v2.5-pro\")\n\nrouter = APIRouter(prefix=\"/api/papers\", tags=[\"summarize\"])\n\n\n# ---------------------------------------------------------------------------\n# Text extraction helpers\n# ---------------------------------------------------------------------------\n\ndef _extract_text_from_file(file_path: str) -> str:\n    \"\"\"Extract text from a local PDF file using pdfplumber.\"\"\"\n    pages_text: list[str] = []\n    with pdfplumber.open(file_path) as pdf:\n        for page in pdf.pages:\n            text = page.extract_text()\n            if text:\n                pages_text.append(text.strip())\n    return \"\\n\\n\".join(pages_text)\n\n\ndef _extract_text_from_bytes(pdf_bytes: bytes) -> str:\n    \"\"\"Extract text from PDF bytes (for R2 or streamed files).\"\"\"\n    pages_text: list[str] = []\n    with tempfile.NamedTemporaryFile(delete=False, suffix=\".pdf\") as tmp:\n        tmp.write(pdf_bytes)\n        tmp_path = tmp.name\n    try:\n        with pdfplumber.open(tmp_path) as pdf:\n            for page in pdf.pages:\n                text = page.extract_text()\n                if text:\n                    pages_text.append(text.strip())\n    finally:\n        os.remove(tmp_path)\n    return \"\\n\\n\".join(pages_text)\n\n\ndef _extract_pages_from_file(file_path: str) -> list[dict]:\n    \"\"\"Extract text per page from a local PDF file.\"\"\"\n    pages: list[dict] = []\n    with pdfplumber.open(file_path) as pdf:\n        for i, page in enumerate(pdf.pages, 1):\n            text = page.extract_text()\n            if text and text.strip():\n                pages.append({\"page\": i, \"text\": text.strip()})\n    return pages\n\n\ndef _extract_pages_from_bytes(pdf_bytes: bytes) -> list[dict]:\n    \"\"\"Extract text per page from PDF bytes.\"\"\"\n    pages: list[dict] = []\n    with tempfile.NamedTemporaryFile(delete=False, suffix=\".pdf\") as tmp:\n        tmp.write(pdf_bytes)\n        tmp_path = tmp.name\n    try:\n        with pdfplumber.open(tmp_path) as pdf:\n            for i, page in enumerate(pdf.pages, 1):\n                text = page.extract_text()\n                if text and text.strip():\n                    pages.append({\"page\": i, \"text\": text.strip()})\n    finally:\n        os.remove(tmp_path)\n    return pages\n\n\n# ---------------------------------------------------------------------------\n# Helper: get PDF text based on paper storage\n# ---------------------------------------------------------------------------\n\ndef _get_paper_pages(paper: Paper) -> list[dict]:\n    \"\"\"Get per-page text from a paper's PDF, handling local/R2/URL sources.\"\"\"\n    if paper.pdf_path:\n        if is_r2_path(paper.pdf_path):\n            chunks = []\n            for chunk in stream_r2_object(paper.pdf_path):\n                chunks.append(chunk)\n            pdf_bytes = b\"\".join(chunks)\n            return _extract_pages_from_bytes(pdf_bytes)\n        elif os.path.exists(paper.pdf_path):\n            return _extract_pages_from_file(paper.pdf_path)\n    elif paper.pdf_url:\n        headers = {\n            \"User-Agent\": BROWSER_USER_AGENT,\n        }\n        with httpx.Client(timeout=60, follow_redirects=True) as client:\n            resp = client.get(paper.pdf_url, headers=headers)\n            resp.raise_for_status()\n            return _extract_pages_from_bytes(resp.content)\n    return []\n\n\n# ---------------------------------------------------------------------------\n# AI Settings Helper\n# ---------------------------------------------------------------------------\n\ndef _get_ai_setting(db: Session, key: str, env_key: str, default: str = \"\") -> str:\n    \"\"\"Read a setting from DB first, fallback to env var.\"\"\"\n    setting = db.query(Setting).filter(Setting.key == key).first()\n    if setting and setting.value:\n        return setting.value\n    return env(env_key, default)\n\n\n# AI Summarization\n# ---------------------------------------------------------------------------\n\ndef _summarize_with_ai(text: str, max_sentences: int = 7, paper_title: str = \"\", lang: str = \"id\", db: Session = None) -> dict:\n    \"\"\"Use AI API to summarize the extracted PDF text.\"\"\"\n    if db:\n        api_key = _get_ai_setting(db, \"ai_api_key\", \"AI_API_KEY\")\n        base_url = _get_ai_setting(db, \"ai_base_url\", \"AI_BASE_URL\")\n        model_name = _get_ai_setting(db, \"ai_model\", \"AI_MODEL\", AI_DEFAULT_MODEL)\n    else:\n        api_key = env(\"AI_API_KEY\")\n        base_url = env(\"AI_BASE_URL\")\n        model_name = env(\"AI_MODEL\", AI_DEFAULT_MODEL)\n\n    if not api_key:\n        raise HTTPException(\n            status_code=500,\n            detail=\"AI_API_KEY not configured. Please set it in environment variables.\",\n        )\n\n    # Truncate text to avoid exceeding context limits (roughly 12000 words)\n    words = text.split()\n    if len(words) > 12000:\n        text = \" \".join(words[:12000]) + \"\\n\\n[...text truncated...]\"\n\n    title_context = f'The paper is titled: \"{paper_title}\".\\n\\n' if paper_title else \"\"\n\n    # Determine response language instruction\n    if lang == \"id\":\n        lang_instruction = \"You MUST respond entirely in Indonesian (Bahasa Indonesia). All summary text and key points must be written in Indonesian.\"\n        system_msg = (\n            \"Anda adalah asisten penelitian akademik. Ringkas paper penelitian dan harus mengandung 1) latar belakang riset, 2) research gap, 3) metodologi, 4) hasil, 5) temuan, 6) kesimpulan\"\n            \"Selalu berikan respons dalam format JSON yang valid sesuai permintaan. \"\n            \"Seluruh respons HARUS dalam Bahasa Indonesia.\"\n        )\n    else:\n        lang_instruction = \"Respond in English.\"\n        system_msg = (\n            \"You are an academic research assistant. Summarize research papers and ensure the summary includes 1) research background, 2) research gap, 3) methodology, 4) results, 5) findings, and 6) conclusions.\"\n            \"Always provide the response in valid JSON format according to the requested schema. The entire response MUST be in English.\"\n        )\n\n    prompt = f\"\"\"{title_context}Please analyze the following academic paper text and provide:\n\n1. A comprehensive summary in approximately {max_sentences} key sentences that captures the research background, gap, methodology, results, findings, and conclusions of the paper.\n2. A list of key points (each as a single concise sentence).\n\n{lang_instruction}\n\nReturn your response in the following JSON format only, with no additional text:\n{{\n  \"summary\": \"Your summary paragraph here...\",\n  \"key_points\": [\"Key point 1\", \"Key point 2\", \"Key point 3\", ...]\n}}\n\nPaper text:\n{text}\"\"\"\n\n    try:\n        from openai import OpenAI\n\n        client = OpenAI(\n            api_key=api_key,\n            base_url=base_url,\n        )\n\n        response = client.chat.completions.create(\n            model=model_name,\n            messages=[\n                {\n                    \"role\": \"system\",\n                    \"content\": system_msg,\n                },\n                {\"role\": \"user\", \"content\": prompt},\n            ],\n            temperature=0.3,\n            max_tokens=2000,\n        )\n\n        content = response.choices[0].message.content.strip()\n\n        # Extract token usage\n        prompt_tokens = 0\n        completion_tokens = 0\n        total_tokens = 0\n        if hasattr(response, \"usage\") and response.usage:\n            prompt_tokens = response.usage.prompt_tokens or 0\n            completion_tokens = response.usage.completion_tokens or 0\n            total_tokens = response.usage.total_tokens or 0\n\n        # Try to parse JSON from response (handle markdown code blocks)\n        # Remove markdown code block markers if present\n        if content.startswith(\"```\"):\n            content = re.sub(r\"^```(?:json)?\\s*\", \"\", content)\n            content = re.sub(r\"\\s*```$\", \"\", content)\n\n        result = json.loads(content)\n\n        summary = result.get(\"summary\", \"\")\n        key_points = result.get(\"key_points\", [])\n\n        if not summary and key_points:\n            summary = \" \".join(key_points)\n\n        return {\n            \"summary\": summary,\n            \"key_points\": key_points,\n            \"word_count_original\": len(text.split()),\n            \"word_count_summary\": len(summary.split()),\n            \"prompt_tokens\": prompt_tokens,\n            \"completion_tokens\": completion_tokens,\n            \"total_tokens\": total_tokens,\n        }\n\n    except json.JSONDecodeError:\n        # If JSON parsing fails, use the raw response as summary\n        return {\n            \"summary\": content,\n            \"key_points\": [content],\n            \"word_count_original\": len(text.split()),\n            \"word_count_summary\": len(content.split()),\n            \"prompt_tokens\": prompt_tokens if \"prompt_tokens\" in dir() else 0,\n            \"completion_tokens\": completion_tokens if \"completion_tokens\" in dir() else 0,\n            \"total_tokens\": total_tokens if \"total_tokens\" in dir() else 0,\n        }\n    except Exception as e:\n        raise HTTPException(\n            status_code=502,\n            detail=f\"AI API error: {str(e)}\",\n        )\n\n\n# ---------------------------------------------------------------------------\n# API Endpoints\n# ---------------------------------------------------------------------------\n\nclass SummarizeRequest(BaseModel):\n    max_sentences: int = 7\n    lang: str = \"id\"\n\n\nclass SummarizeResponse(BaseModel):\n    summary: str\n    key_points: list[str]\n    word_count_original: int\n    word_count_summary: int\n    pages_processed: int\n    total_pages: int\n    prompt_tokens: int = 0\n    completion_tokens: int = 0\n    total_tokens: int = 0\n\n\nclass PageTextResponse(BaseModel):\n    page: int\n    text: str\n\n\nclass ExtractTextResponse(BaseModel):\n    pages: list[PageTextResponse]\n    total_pages: int\n\n\nclass AutoHighlightSentence(BaseModel):\n    text: str\n    text_id: str = \"\"  # Indonesian translation\n    page: int\n    category: str  # e.g. \"metodologi\", \"temuan\", \"kontribusi\", \"latar_belakang\", \"kesimpulan\"\n\n\nclass AutoHighlightResponse(BaseModel):\n    sentences: list[AutoHighlightSentence]\n    total_found: int\n    pages_scanned: int\n\n\n@router.get(\"/{paper_id}/summarize\", response_model=SummarizeResponse)\ndef summarize_paper(\n    paper_id: int,\n    max_sentences: int = 7,\n    lang: str = \"id\",\n    force: bool = False,\n    current_user: User = Depends(get_current_user),\n    db: Session = Depends(get_db),\n):\n    \"\"\"Extract and summarize the text content of a paper's PDF using AI.\n    Results are cached in the database. Use force=true to regenerate.\"\"\"\n    paper = db.query(Paper).filter(\n        Paper.id == paper_id,\n        Paper.user_id == current_user.id,\n    ).first()\n    if not paper:\n        raise HTTPException(status_code=404, detail=\"Paper not found\")\n\n    # Check for cached summary (same language)\n    if not force and paper.summary_text and paper.summary_lang == lang:\n        try:\n            key_points = json.loads(paper.summary_key_points) if paper.summary_key_points else []\n        except json.JSONDecodeError:\n            key_points = []\n\n        return {\n            \"summary\": paper.summary_text,\n            \"key_points\": key_points,\n            \"word_count_original\": len(paper.summary_text.split()),\n            \"word_count_summary\": len(paper.summary_text.split()),\n            \"pages_processed\": paper.summary_pages,\n            \"total_pages\": paper.summary_pages,\n        }\n\n    # Generate new summary\n    pages = _get_paper_pages(paper)\n\n    if not pages:\n        raise HTTPException(\n            status_code=422,\n            detail=\"Could not extract text from the PDF. The file may be image-based or corrupted.\",\n        )\n\n    full_text = \"\\n\\n\".join(p[\"text\"] for p in pages)\n    total_pages = len(pages)\n\n    result = _summarize_with_ai(\n        full_text,\n        max_sentences=max_sentences,\n        paper_title=paper.title,\n        lang=lang,\n        db=db,\n    )\n    result[\"pages_processed\"] = total_pages\n    result[\"total_pages\"] = total_pages\n\n    # Cache result in database\n    paper.summary_text = result[\"summary\"]\n    paper.summary_key_points = json.dumps(result[\"key_points\"], ensure_ascii=False)\n    paper.summary_lang = lang\n    paper.summary_pages = total_pages\n    paper.summary_generated_at = datetime.now(timezone.utc)\n\n    # Log token usage\n    logged_model = _get_ai_setting(db, \"ai_model\", \"AI_MODEL\", AI_DEFAULT_MODEL)\n    usage_log = AIUsageLog(\n        user_id=current_user.id,\n        paper_id=paper.id,\n        feature=\"summarize\",\n        model=logged_model,\n        prompt_tokens=result.get(\"prompt_tokens\", 0),\n        completion_tokens=result.get(\"completion_tokens\", 0),\n        total_tokens=result.get(\"total_tokens\", 0),\n    )\n    db.add(usage_log)\n    db.commit()\n\n    return result\n\n\n@router.delete(\"/{paper_id}/summary\")\ndef delete_summary(\n    paper_id: int,\n    current_user: User = Depends(get_current_user),\n    db: Session = Depends(get_db),\n):\n    \"\"\"Delete cached summary for a paper.\"\"\"\n    paper = db.query(Paper).filter(\n        Paper.id == paper_id,\n        Paper.user_id == current_user.id,\n    ).first()\n    if not paper:\n        raise HTTPException(status_code=404, detail=\"Paper not found\")\n\n    paper.summary_text = \"\"\n    paper.summary_key_points = \"\"\n    paper.summary_lang = \"\"\n    paper.summary_pages = 0\n    paper.summary_generated_at = None\n    db.commit()\n\n    return {\"message\": \"Summary deleted\"}\n\n\n@router.get(\"/{paper_id}/extract-text\", response_model=ExtractTextResponse)\ndef extract_paper_text(\n    paper_id: int,\n    current_user: User = Depends(get_current_user),\n    db: Session = Depends(get_db),\n):\n    \"\"\"Extract per-page text from a paper's PDF.\"\"\"\n    paper = db.query(Paper).filter(\n        Paper.id == paper_id,\n        Paper.user_id == current_user.id,\n    ).first()\n    if not paper:\n        raise HTTPException(status_code=404, detail=\"Paper not found\")\n\n    pages = _get_paper_pages(paper)\n    return {\"pages\": pages, \"total_pages\": len(pages)}\n\n\n# ---------------------------------------------------------------------------\n# Auto-highlight with AI\n# ---------------------------------------------------------------------------\n\ndef _generate_auto_highlights_with_ai(\n    pages: list[dict], paper_title: str, lang: str = \"id\", max_sentences: int = 15, query: str = \"\", db: Session = None\n) -> list[dict]:\n    \"\"\"Use AI to identify key sentences relevant to the research topic.\n    \n    Args:\n        pages: List of page texts from the PDF\n        paper_title: The paper's title (used as fallback context)\n        lang: Language code (\"id\" or \"en\")\n        max_sentences: Maximum number of sentences to return\n        query: Custom topic/title/reference sentence to search for (optional)\n    \"\"\"\n    if db:\n        api_key = _get_ai_setting(db, \"ai_api_key\", \"AI_API_KEY\")\n        base_url = _get_ai_setting(db, \"ai_base_url\", \"AI_BASE_URL\")\n        model_name = _get_ai_setting(db, \"ai_model\", \"AI_MODEL\", AI_DEFAULT_MODEL)\n    else:\n        api_key = env(\"AI_API_KEY\")\n        base_url = env(\"AI_BASE_URL\")\n        model_name = env(\"AI_MODEL\", AI_DEFAULT_MODEL)\n\n    if not api_key:\n        raise HTTPException(\n            status_code=500,\n            detail=\"AI_API_KEY not configured. Please set it in environment variables.\",\n        )\n\n    # Build page-tagged text (limit to avoid context overflow)\n    page_texts = []\n    word_count = 0\n    for p in pages:\n        page_words = p[\"text\"].split()\n        if word_count + len(page_words) > 12000:\n            remaining = 12000 - word_count\n            if remaining > 50:\n                page_texts.append(f\"[PAGE {p['page']}]\\n{' '.join(page_words[:remaining])}\")\n            break\n        page_texts.append(f\"[PAGE {p['page']}]\\n{p['text']}\")\n        word_count += len(page_words)\n\n    tagged_text = \"\\n\\n\".join(page_texts)\n\n    if lang == \"id\":\n        system_msg = (\n            \"Anda adalah asisten penelitian akademik. Tugas Anda adalah mengidentifikasi \"\n            \"kalimat-kalimat penting dan relevan dari paper penelitian berdasarkan judul dan topiknya. \"\n            \"Fokus pada kalimat yang mengandung: 1) latar belakang/riset gap, 2) kontribusi utama, \"\n            \"3) metodologi kunci, 4) temuan/result utama, 5) kesimpulan penting. \"\n            \"Kalimat harus diambil persis dari teks asli, jangan diubah atau diparafrase. \"\n            \"Selalu berikan respons dalam format JSON yang valid. \"\n            \"Seluruh label kategori HARUS dalam Bahasa Indonesia.\"\n        )\n        lang_instruction = \"Seluruh respons HARUS dalam Bahasa Indonesia.\"\n        category_labels = \"latar_belakang, kontribusi, metodologi, temuan, kesimpulan\"\n    else:\n        system_msg = (\n            \"You are an academic research assistant. Your task is to identify key and relevant \"\n            \"sentences from a research paper based on its title and topic. Focus on sentences that \"\n            \"contain: 1) background/research gap, 2) main contributions, 3) key methodology, \"\n            \"4) main findings/results, 5) important conclusions. \"\n            \"Sentences must be taken exactly from the original text, do not change or paraphrase them. \"\n            \"Always provide the response in valid JSON format.\"\n        )\n        lang_instruction = \"Respond in English.\"\n        category_labels = \"background, contribution, methodology, finding, conclusion\"\n\n    # Build context based on whether custom query is provided\n    if query:\n        if lang == \"id\":\n            search_context = (\n                f'Paper ini berjudul: \"{paper_title}\"\\n\\n'\n                f'TOPIC/KALIMAT REFERENSI YANG DICARI: \"{query}\"\\n\\n'\n                f'Analisis paper berikut dan identifikasi maksimal {max_sentences} kalimat yang relevan '\n                f'dengan topik/kalimat referensi di atas. Untuk setiap kalimat, sebutkan halaman dan kategorinya.'\n            )\n        else:\n            search_context = (\n                f'The paper is titled: \"{paper_title}\"\\n\\n'\n                f'TOPIC/REFERENCE SENTENCE TO SEARCH FOR: \"{query}\"\\n\\n'\n                f'Analyze the following paper and identify up to {max_sentences} key sentences that are relevant '\n                f'to the topic/reference above. For each sentence, specify which page it is on and categorize it.'\n            )\n    else:\n        search_context = (\n            f'The paper is titled: \"{paper_title}\"\\n\\n'\n            f'Analyze the following paper text and identify up to {max_sentences} key sentences '\n            f'that are most relevant to the research topic/title. For each sentence, specify which page it is on and categorize it.'\n        )\n\n    prompt = f\"\"\"{search_context}\n\n{lang_instruction}\n\nAvailable categories: {category_labels}\n\nIMPORTANT RULES:\n1. Each sentence MUST be an EXACT quote from the paper text (do not modify or paraphrase)\n2. Each sentence should be a complete, meaningful sentence (not fragments)\n3. Prioritize the most important/relevant sentences\n4. Include the page number where each sentence appears (from the [PAGE N] tags)\n5. For each sentence, provide an Indonesian translation in the \"text_id\" field\n\nReturn ONLY valid JSON in this format:\n{{\n  \"sentences\": [\n    {{\"text\": \"exact sentence from paper\", \"text_id\": \"terjemahan dalam bahasa Indonesia\", \"page\": 1, \"category\": \"category_name\"}},\n    {{\"text\": \"another sentence\", \"text_id\": \"terjemahan lain dalam bahasa Indonesia\", \"page\": 3, \"category\": \"category_name\"}}\n  ]\n}}\n\nPaper text:\n{tagged_text}\"\"\"\n\n    try:\n        from openai import OpenAI\n\n        client = OpenAI(api_key=api_key, base_url=base_url)\n\n        response = client.chat.completions.create(\n            model=model_name,\n            messages=[\n                {\"role\": \"system\", \"content\": system_msg},\n                {\"role\": \"user\", \"content\": prompt},\n            ],\n            temperature=0.3,\n            max_tokens=3000,\n        )\n\n        content = response.choices[0].message.content.strip()\n\n        # Remove markdown code block markers if present\n        if content.startswith(\"```\"):\n            content = re.sub(r\"^```(?:json)?\\s*\", \"\", content)\n            content = re.sub(r\"\\s*```$\", \"\", content)\n\n        result = json.loads(content)\n        sentences = result.get(\"sentences\", [])\n\n        # Validate each sentence\n        validated = []\n        for s in sentences:\n            text = (s.get(\"text\") or \"\").strip()\n            text_id = (s.get(\"text_id\") or \"\").strip()\n            page = s.get(\"page\", 1)\n            category = (s.get(\"category\") or \"lainnya\").strip()\n            if text and len(text) > 10:  # minimum reasonable sentence length\n                validated.append({\"text\": text, \"text_id\": text_id, \"page\": page, \"category\": category})\n\n        # Extract token usage\n        prompt_tokens = 0\n        completion_tokens = 0\n        total_tokens = 0\n        if hasattr(response, \"usage\") and response.usage:\n            prompt_tokens = response.usage.prompt_tokens or 0\n            completion_tokens = response.usage.completion_tokens or 0\n            total_tokens = response.usage.total_tokens or 0\n\n        return validated, {\"prompt_tokens\": prompt_tokens, \"completion_tokens\": completion_tokens, \"total_tokens\": total_tokens}\n\n    except json.JSONDecodeError:\n        # If JSON parsing fails, return empty\n        return [], {\"prompt_tokens\": 0, \"completion_tokens\": 0, \"total_tokens\": 0}\n    except Exception as e:\n        raise HTTPException(\n            status_code=502,\n            detail=f\"AI API error: {str(e)}\",\n        )\n\n\n@router.get(\"/{paper_id}/auto-highlights\", response_model=AutoHighlightResponse)\ndef auto_highlight_paper(\n    paper_id: int,\n    lang: str = \"id\",\n    max_sentences: int = 15,\n    query: str = \"\",\n    current_user: User = Depends(get_current_user),\n    db: Session = Depends(get_db),\n):\n    \"\"\"Identify key sentences in a paper relevant to its topic using AI.\n    Returns sentences with page numbers and categories for auto-highlighting.\n    Use `query` to specify a custom topic, research title, or reference sentence to search for.\"\"\"\n    paper = db.query(Paper).filter(\n        Paper.id == paper_id,\n        Paper.user_id == current_user.id,\n    ).first()\n    if not paper:\n        raise HTTPException(status_code=404, detail=\"Paper not found\")\n\n    pages = _get_paper_pages(paper)\n\n    if not pages:\n        raise HTTPException(\n            status_code=422,\n            detail=\"Could not extract text from the PDF. The file may be image-based or corrupted.\",\n        )\n\n    sentences, token_info = _generate_auto_highlights_with_ai(\n        pages,\n        paper_title=paper.title,\n        lang=lang,\n        max_sentences=max_sentences,\n        query=query.strip() if query else \"\",\n        db=db,\n    )\n\n    # Log token usage\n    logged_model = _get_ai_setting(db, \"ai_model\", \"AI_MODEL\", AI_DEFAULT_MODEL)\n    usage_log = AIUsageLog(\n        user_id=current_user.id,\n        paper_id=paper.id,\n        feature=\"auto_highlight\",\n        model=logged_model,\n        prompt_tokens=token_info.get(\"prompt_tokens\", 0),\n        completion_tokens=token_info.get(\"completion_tokens\", 0),\n        total_tokens=token_info.get(\"total_tokens\", 0),\n    )\n    db.add(usage_log)\n    db.commit()\n\n    return {\n        \"sentences\": sentences,\n        \"total_found\": len(sentences),\n        \"pages_scanned\": len(pages),\n    }"}}}}