{"repo": "aurumz-rgb/reviewaid", "files": {"utils.py": {"26972ab32c8156da585b28d981b5e8ef23721a6e": {"t": 1785674174.9680982, "n": 33646, "h": "8f9300f0daddf937", "c": "import streamlit as st\nimport fitz \nimport time\nimport os \nimport re\nimport json\nimport json5\nimport pandas as pd\nimport io\nimport html\nimport hashlib\nimport gc\nfrom abc import ABC, abstractmethod\nfrom typing import List, Dict, Any\nfrom datetime import datetime\nfrom dotenv import load_dotenv\nimport firebase_admin\nfrom firebase_admin import credentials, firestore\nfrom cryptography.fernet import Fernet\nfrom fpdf import FPDF\nfrom streamlit_lottie import st_lottie\nimport streamlit.components.v1 as components\nimport base64\nimport pytesseract\nfrom PIL import Image\n\ntry:\n from openai import OpenAI\nexcept ImportError:\n OpenAI = None\n\ntry:\n from anthropic import Anthropic\nexcept ImportError:\n Anthropic = None\n\ntry:\n import cohere\nexcept ImportError:\n cohere = None\n\ntry:\n import ollama\nexcept ImportError:\n ollama = None\n\ntry:\n from zai import ZaiClient\nexcept ImportError:\n ZaiClient = None\n\nload_dotenv()\n\n\ndb = None \n\ntry:\n if not firebase_admin._apps:\n firebase_key_str = os.getenv(\"FIREBASE_KEY\")\n if firebase_key_str:\n cred_dict = json.loads(firebase_key_str)\n cred = credentials.Certificate(cred_dict)\n firebase_admin.initialize_app(cred)\n \n if firebase_admin._apps:\n db = firestore.client()\nexcept Exception as e:\n \n try:\n st.warning(f\"Error initializing Firebase: {e}\")\n except:\n print(f\"Error initializing Firebase: {e}\")\n\nMAX_LOG_ENTRIES = 200\nMAX_INPUT_TOKENS_SCREENER = 128000 \nMAX_INPUT_TOKENS_EXTRACTOR = 128000 \nMAX_OUTPUT_TOKENS = 8192\n\n\ndef get_firebase_stats():\n if db is None:\n return {\"papers_screened\": 0, \"papers_extracted\": 0}\n try:\n doc_ref = db.collection(\"ReviewAidAnalytics\").document(\"counters\")\n doc = doc_ref.get()\n \n if doc:\n return {\n \"papers_screened\": doc.to_dict().get(\"papers_screened\", 0),\n \"papers_extracted\": doc.to_dict().get(\"papers_extracted\", 0),\n \n }\n else:\n \n return {\"papers_screened\": 0, \"papers_extracted\": 0}\n except Exception:\n \n return {\"papers_screened\": 0, \"papers_extracted\": 0}\n\ndef display_citation_section():\n \"\"\"\n Displays the citation section.\n Checks session state to prevent duplicate element IDs on re-runs.\n \"\"\"\n\n\n st.markdown(\"---\")\n st.markdown(\"## Citation\")\n st.markdown(\"You can check ReviewAid's journal paper here: [Journal of Open Research Software](https://doi.org/10.5334/jors.672). If you are to use ReviewAid, please don't forget to cite the tool:\")\n\n apa_citation = (\n \"Sahu, V., & Balakrishnan, M. (2026). ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews. \"\n \"Journal of Open Research Software, 14(1), 21. https://doi.org/10.5334/jors.672\"\n )\n\n harvard_citation = (\n \"Sahu, V. and Balakrishnan, M., 2026. ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews. \"\n \"Journal of Open Research Software, 14(1), p. 21. Available at: https://doi.org/10.5334/jors.672\"\n )\n\n mla_citation = (\n \"Sahu, Vihaan, and Mohith Balakrishnan. \\\"ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews.\\\" \"\n \"Journal of Open Research Software, vol. 14, no. 1, 2026, p. 21.\"\n )\n\n chicago_citation = (\n \"Sahu, Vihaan, and Mohith Balakrishnan. 2026. \\\"ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews.\\\" \"\n \"Journal of Open Research Software 14 (1): 21. https://doi.org/10.5334/jors.672.\"\n )\n\n ieee_citation = (\n \"V. Sahu and M. Balakrishnan, \\\"ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews,\\\" \"\n \"Journal of Open Research Software, vol. 14, no. 1, p. 21, 2026. doi: 10.5334/jors.672.\"\n )\n\n vancouver_citation = (\n \"Sahu V, Balakrishnan M. ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews. \"\n \"J Open Res Softw. 2026;14(1):21. doi:10.5334/jors.672\"\n )\n\n ris_data = \"\"\"TY - JOUR\nAU - Sahu, V.\nAU - Balakrishnan, M.\nTI - ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews\nJO - Journal of Open Research Software\nPY - 2026\nVL - 14\nIS - 1\nSP - 21\nDO - 10.5334/jors.672\nER -\"\"\"\n\n bib_data = \"\"\"@article{Sahu2026,\n author = {Sahu, V. and Balakrishnan, M.},\n title = {ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews},\n journal = {Journal of Open Research Software},\n year = {2026},\n volume = {14},\n number = {1},\n pages = {21},\n doi = {10.5334/jors.672}\n}\"\"\"\n \n try:\n citation_style = st.selectbox(\n \"Select citation style\",\n [\"APA\", \"Harvard\", \"MLA\", \"Chicago\", \"IEEE\", \"Vancouver\"]\n )\n except st.errors.StreamlitDuplicateElementId as e:\n st.warning(f\"Citation section skipped (Duplicate ID): {e}\")\n \n return\n\n if citation_style == \"APA\":\n citation_text = apa_citation\n elif citation_style == \"Harvard\":\n citation_text = harvard_citation\n elif citation_style == \"MLA\":\n citation_text = mla_citation\n elif citation_style == \"Chicago\":\n citation_text = chicago_citation\n elif citation_style == \"IEEE\":\n citation_text = ieee_citation\n elif citation_style == \"Vancouver\":\n citation_text = vancouver_citation\n\n escaped_citation = html.escape(citation_text)\n \n st.markdown(f'

{escaped_citation}

', unsafe_allow_html=True)\n\n js_citation_text = json.dumps(citation_text)\n \n st.markdown(f\"\"\"\n
\n \n RIS Format\n BibTeX Format\n
\n \"\"\", unsafe_allow_html=True)\n\n st.markdown(f\"\"\"\n \n \"\"\", unsafe_allow_html=True)\n\n\n\ndef init_analytics():\n pass\n\n\ndef increment_firebase_counter(field):\n if db is None: return\n try:\n doc_ref = db.collection(\"ReviewAidAnalytics\").document(\"counters\")\n doc_ref.update({field: firestore.Increment(1)})\n except Exception as e: print(f\"Analytics Error: {e}\")\n\ndef update_processing_stats(mode, count=1):\n if db is None: return\n if mode == \"screener\":\n for _ in range(count): increment_firebase_counter(\"papers_screened\")\n elif mode == \"extractor\":\n for _ in range(count): increment_firebase_counter(\"papers_extracted\")\n\ndef update_terminal_log(msg, level=\"INFO\"):\n if \"terminal_logs\" not in st.session_state or \"terminal_placeholder\" not in st.session_state:\n return\n\n timestamp = datetime.now().strftime(\"%H:%M:%S.%f\")[:-3] \n \n if level == \"INFO\":\n color_class = \"log-info\"\n elif level == \"SUCCESS\":\n color_class = \"log-success\"\n elif level == \"WARN\":\n color_class = \"log-warn\"\n elif level == \"ERROR\":\n color_class = \"log-error\"\n elif level == \"SYSTEM\":\n color_class = \"log-system\"\n elif level == \"DEBUG\":\n color_class = \"log-debug\"\n else:\n color_class = \"log-info\"\n \n log_entry = f'[{timestamp}]{level}: {html.escape(msg)}'\n \n st.session_state.terminal_logs.append(log_entry)\n \n if len(st.session_state.terminal_logs) > MAX_LOG_ENTRIES:\n st.session_state.terminal_logs.pop(0)\n \n current_time = time.time()\n if current_time - st.session_state.last_log_update_time > 0.5:\n full_log_html = '
' + \"\".join(st.session_state.terminal_logs) + '
'\n scroll_script = \"\"\"\n \n \"\"\"\n try:\n st.session_state.terminal_placeholder.markdown(full_log_html + scroll_script, unsafe_allow_html=True)\n except Exception:\n pass\n st.session_state.last_log_update_time = current_time\n\ndef extract_pdf_content(pdf_bytes, enable_ocr=False):\n try:\n update_terminal_log(\"Initializing PDF extraction engine (PyMuPDF)...\", \"DEBUG\")\n except:\n pass\n \n doc = None\n try:\n doc = fitz.open(stream=pdf_bytes, filetype=\"pdf\")\n page_count = doc.page_count\n try:\n update_terminal_log(f\"Document opened successfully. Total pages: {page_count}\", \"INFO\")\n except:\n pass\n \n MAX_CHAR_LIMIT = 600000 \n full_text_parts = []\n current_length = 0\n references_found = False\n \n \n ocr_text_parts = []\n total_images_found = 0\n \n for i, page in enumerate(doc):\n if current_length > MAX_CHAR_LIMIT and not references_found:\n try:\n update_terminal_log(f\"Token limit reached at Page {i+1}. Stopping PDF read.\", \"INFO\")\n except:\n pass\n break\n \n page_text = page.get_text()\n \n if re.search(r'(?:\\n|\\r\\n){1,2}(References|Reference|Bibliography)(?:\\s|\\r?\\n|$)', page_text, re.IGNORECASE):\n references_found = True \n\n full_text_parts.append(page_text)\n current_length += len(page_text)\n \n \n if enable_ocr:\n try:\n image_list = page.get_images(full=True)\n if image_list:\n num_images_on_page = len(image_list)\n update_terminal_log(f\"Page {i+1}: Found {num_images_on_page} image(s). Running Tesseract OCR...\", \"INFO\")\n \n page_ocr_success = False\n \n for img in image_list:\n try:\n xref = img[0]\n base_image = doc.extract_image(xref)\n image_bytes = base_image[\"image\"]\n image = Image.open(io.BytesIO(image_bytes))\n \n \n extracted_text = pytesseract.image_to_string(image)\n \n if extracted_text.strip():\n ocr_text_parts.append(extracted_text)\n page_ocr_success = True\n \n \n del image, image_bytes\n \n except Exception:\n pass \n \n if page_ocr_success:\n update_terminal_log(f\"Page {i+1}: OCR complete.\", \"SUCCESS\")\n \n total_images_found += num_images_on_page\n except Exception as e:\n update_terminal_log(f\"Page {i+1}: Error scanning for images - {str(e)}\", \"ERROR\")\n \n \n del page_text \n \n full_text = \"\".join(full_text_parts)\n del full_text_parts\n \n\n \n if enable_ocr:\n if total_images_found > 0:\n ocr_content = \"\\n\\n\".join(ocr_text_parts)\n full_text += \"\\n\\n[OCR EXTRACTED CONTENT]:\\n\" + ocr_content\n update_terminal_log(f\"Finished. Total images OCR'd: {total_images_found} (Data Extracted from Images)\", \"SUCCESS\")\n else:\n update_terminal_log(\"OCR Enabled, but no images were found in this document.\", \"WARN\")\n\n\n ref_match = re.search(r'(?:\\n|\\r\\n){1,2}(References|Reference|Bibliography)(?:\\s|\\r?\\n|$)', full_text, re.IGNORECASE)\n \n if ref_match:\n main_text_end = ref_match.start()\n full_text = full_text[:main_text_end]\n try:\n update_terminal_log(\"References section detected and removed to save tokens.\", \"INFO\")\n except:\n pass\n del ref_match\n\n metadata = doc.metadata\n title = metadata.get('title', '')\n author = metadata.get('author', '')\n del metadata \n \n try:\n update_terminal_log(f\"Metadata read -> Title: '{title}', Author: '{author}'\", \"DEBUG\")\n except:\n pass\n \n if not title or len(title) < 5 or \"microsoft\" in title.lower() or \".doc\" in title.lower() or title.isdigit():\n try:\n update_terminal_log(\"Metadata title appears empty or invalid. Scanning first page text for title...\", \"DEBUG\")\n except:\n pass\n \n first_page_text = full_text[:3000]\n lines = first_page_text.split('\\n')\n \n for line in lines:\n clean_line = line.strip()\n\n if len(clean_line) > 10 and len(clean_line) < 200:\n if not re.match(r'^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}$', clean_line, re.IGNORECASE) and \\\n not clean_line.startswith(\"http\") and \\\n \"abstract\" not in clean_line.lower() and \\\n \"introduction\" not in clean_line.lower() and \\\n \"keywords\" not in clean_line.lower() and \\\n \"page\" not in clean_line.lower() and \\\n \"vol.\" not in clean_line.lower():\n \n\n title = clean_line\n try:\n update_terminal_log(f\"Found candidate title in text: '{title[:50]}...'\", \"INFO\")\n except:\n pass\n break\n \n year = ''\n if not year:\n if page_count > 0:\n year_match = re.search(r'\\b(19|20)\\d{2}\\b', full_text[:5000]) \n if year_match:\n year = year_match.group()\n try:\n update_terminal_log(f\"Year found on Page 1: {year}\", \"DEBUG\")\n except:\n pass\n \n if not year:\n creation_date = doc.metadata.get('creationDate', '')\n if creation_date:\n year_match = re.search(r'\\b(19|20)\\d{2}\\b', creation_date)\n if year_match:\n year = year_match.group()\n try:\n update_terminal_log(f\"Year derived from creationDate: {year}\", \"DEBUG\")\n except:\n pass\n \n # Perform garbage collection if OCR was used to make space to save RAM\n if enable_ocr and total_images_found > 0:\n gc.collect()\n update_terminal_log(\"OCR'd images will be cleared to save space.\", \"SYSTEM\")\n\n return full_text, title, author, year\n \n except Exception as e:\n try:\n update_terminal_log(f\"Error during PDF extraction: {str(e)}\", \"ERROR\")\n except:\n pass\n return \"\", \"\", \"\", \"\"\n finally:\n if doc:\n doc.close()\n del doc\n\ndef preprocess_text_for_ai(text, max_tokens=MAX_INPUT_TOKENS_SCREENER):\n if \" \" in text or \"\\n\" in text:\n text = \" \".join(text.split())\n \n char_limit = max_tokens * 4\n if len(text) > char_limit:\n try:\n update_terminal_log(f\"Text exceeds token limit ({len(text)} > {char_limit}). Truncating...\", \"WARN\")\n except:\n pass\n text = text[:char_limit]\n \n return text\n\ndef to_docx(df):\n from docx import Document\n from docx.shared import Inches, Pt, RGBColor\n from docx.enum.text import WD_ALIGN_PARAGRAPH\n from docx.oxml.ns import qn\n from docx.oxml import OxmlElement\n\n doc = Document()\n \n sections = doc.sections\n for section in sections:\n section.top_margin = Inches(0.5)\n section.bottom_margin = Inches(0.5) \n section.left_margin = Inches(0.5)\n section.right_margin = Inches(0.5)\n\n section = doc.sections[0]\n \n section.page_width = Inches(11.69)\n section.page_height = Inches(8.27)\n\n header = section.header\n header_para = header.paragraphs[0]\n header_para.alignment = WD_ALIGN_PARAGRAPH.RIGHT\n \n logo_path = os.path.join(\"assets\", \"RA_transparent.png\")\n if os.path.exists(logo_path):\n try:\n run_logo = header_para.add_run()\n run_logo.add_picture(logo_path, height=Inches(0.4))\n except Exception:\n pass \n\n footer = section.footer\n footer_para = footer.paragraphs[0]\n footer_para.alignment = WD_ALIGN_PARAGRAPH.LEFT\n \n generated_date = datetime.now().strftime(\"%B %d, %Y\")\n \n footer_text = f\"Generated by ReviewAid on {generated_date}\"\n \n footer_run = footer_para.add_run(footer_text)\n footer_run.font.name = 'Arial'\n footer_run.font.size = Pt(8)\n footer_run.font.color.rgb = RGBColor(100, 100, 100)\n\n title = doc.add_heading('Results', 0)\n title.alignment = WD_ALIGN_PARAGRAPH.CENTER\n \n num_cols = len(df.columns)\n table = doc.add_table(rows=1, cols=num_cols)\n\n table.style = 'Table Grid' \n \n table.allow_autofit = True\n for row in table.rows:\n row.allow_break_across_pages = True\n\n usable_width = section.page_width - section.left_margin - section.right_margin - Inches(0.2)\n col_width = usable_width / num_cols if num_cols > 0 else usable_width\n\n hdr_cells = table.rows[0].cells\n for i, col in enumerate(df.columns):\n cell = hdr_cells[i]\n cell.text = str(col)\n cell.width = col_width \n \n cell_par = cell.paragraphs[0]\n cell_par.runs[0].font.bold = True\n cell_par.runs[0].font.name = 'Arial'\n cell_par.runs[0].font.size = Pt(10)\n cell_par.runs[0].font.color.rgb = RGBColor(0, 0, 0) \n cell_par.alignment = WD_ALIGN_PARAGRAPH.CENTER\n \n shading_elm = OxmlElement('w:shd')\n shading_elm.set(qn('w:fill'), 'FFFFFF') \n cell._element.get_or_add_tcPr().append(shading_elm)\n \n tc_borders = OxmlElement('w:tcBorders')\n bottom = OxmlElement('w:bottom')\n bottom.set(qn('w:val'), 'single')\n bottom.set(qn('w:sz'), '4')\n bottom.set(qn('w:space'), '0')\n bottom.set(qn('w:color'), '000000') \n tc_borders.append(bottom)\n cell._element.get_or_add_tcPr().append(tc_borders)\n\n for _, row in df.iterrows():\n row_cells = table.add_row().cells\n for i, val in enumerate(row):\n cell = row_cells[i]\n cell.width = col_width \n \n para = cell.paragraphs[0]\n \n safe_val = str(val)\n safe_val = re.sub(r'[\\x00-\\x08\\x0b-\\x0c\\x0e-\\x1f\\x7f-\\x9f]', '', safe_val)\n \n run = para.add_run(safe_val)\n run.font.name = 'Arial'\n run.font.size = Pt(9)\n run.font.color.rgb = RGBColor(0, 0, 0) \n \n tc = cell._tc\n tcPr = tc.get_or_add_tcPr()\n shading_elm = OxmlElement('w:shd')\n shading_elm.set(qn('w:fill'), 'FFFFFF')\n tcPr.append(shading_elm)\n\n buffer = io.BytesIO()\n doc.save(buffer)\n buffer.seek(0)\n return buffer.getvalue()\n\ndef to_pdf(df):\n pdf = FPDF(orientation='L', unit='mm', format='A4')\n pdf.add_page()\n \n pdf.set_auto_page_break(auto=True, margin=15) \n \n pdf.set_font(\"Arial\", size=8)\n\n logo_path = os.path.join(\"assets\", \"RA_transparent.png\")\n if os.path.exists(logo_path):\n try:\n pdf.image(logo_path, x=250, y=10, w=30)\n except:\n pass \n\n generated_date = datetime.now().strftime(\"%Y-%m-%d\")\n\n col_width = 277 / len(df.columns) \n line_height = 5 \n\n pdf.set_fill_color(65, 137, 220) \n pdf.set_text_color(255, 255, 255)\n pdf.set_font(\"Arial\", style='B', size=8)\n \n if pdf.get_y() > 260: \n pdf.add_page()\n \n for col in df.columns:\n pdf.cell(col_width, 7, str(col), border=1, align='C', fill=True)\n pdf.ln(7)\n \n pdf.set_text_color(0, 0, 0)\n pdf.set_font(\"Arial\", size=7)\n is_odd = True\n \n table_x_start = pdf.get_x()\n \n for index, row in df.iterrows():\n\n if pdf.get_y() > 250:\n pdf.add_page()\n\n pdf.set_y(10)\n pdf.set_fill_color(65, 137, 220) \n pdf.set_text_color(255, 255, 255)\n pdf.set_font(\"Arial\", style='B', size=8)\n for col in df.columns:\n pdf.cell(col_width, 7, str(col), border=1, align='C', fill=True)\n pdf.ln(7)\n \n pdf.set_text_color(0, 0, 0)\n pdf.set_font(\"Arial\", size=7)\n is_odd = True\n table_x_start = pdf.get_x()\n\n if not is_odd:\n pdf.set_fill_color(245, 245, 245)\n else:\n pdf.set_fill_color(255, 255, 255)\n\n y_start = pdf.get_y()\n max_y_end = y_start\n \n col_heights = []\n for col_name, val in zip(df.columns, row):\n clean_text = str(val).encode('latin-1', 'replace').decode('latin-1')\n \n chars_per_line = int(col_width * 3.5) \n lines = len(clean_text) // chars_per_line + 1\n \n lines += clean_text.count('\\n')\n \n cell_h = lines * line_height\n col_heights.append(cell_h)\n if cell_h > (max_y_end - y_start):\n max_y_end = y_start + cell_h\n \n\n row_height = max_y_end - y_start\n if row_height < line_height:\n row_height = line_height\n\n x_cursor = table_x_start\n \n for i, val in enumerate(row):\n clean_text = str(val).encode('latin-1', 'replace').decode('latin-1')\n\n pdf.set_xy(x_cursor, y_start)\n \n pdf.multi_cell(col_width, row_height, clean_text, border=1, fill=(not is_odd), align='L')\n \n x_cursor += col_width\n\n pdf.set_y(max_y_end)\n pdf.set_x(table_x_start) \n \n is_odd = not is_odd\n\n pdf.set_y(-15)\n pdf.set_font(\"Arial\", size=7, style='I')\n pdf.set_text_color(128, 128, 128)\n pdf.set_x(10) \n pdf.cell(0, 10, f\"Generated by ReviewAid on {generated_date}\", 0, 0, 'L')\n \n return pdf.output(dest='S').encode('latin1')\n\ndef to_csv(df):\n buffer = io.StringIO()\n buffer.write(\"# Generated by ReviewAid\\n\")\n df.to_csv(buffer, index=False)\n return buffer.getvalue().encode('utf-8')\n\ndef to_excel(df):\n import xlsxwriter\n buffer = io.BytesIO()\n with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer:\n df.to_excel(writer, index=False, sheet_name='ReviewAid Data')\n \n workbook = writer.book\n worksheet = writer.sheets['ReviewAid Data']\n \n header_format = workbook.add_format({\n 'bold': True,\n 'text_wrap': True,\n 'valign': 'top',\n 'fg_color': '#4189DC', \n 'font_color': '#FFFFFF',\n 'border': 1\n })\n \n for col_num, value in enumerate(df.columns.values):\n worksheet.write(0, col_num, value, header_format)\n \n generated_date = datetime.now().strftime(\"%Y-%m-%d\")\n\n worksheet.set_footer(f'&LGenerated by ReviewAid on {generated_date}')\n \n buffer.seek(0)\n return buffer.getvalue()\n\n\n\nclass BaseLLMProvider(ABC):\n \"\"\"Abstract base class for LLM providers.\"\"\"\n def __init__(self, api_key: str, model_name: str, **kwargs):\n self.api_key = api_key\n self.model_name = model_name\n self.extra_params = kwargs\n\n @abstractmethod\n def generate(self, messages: List[Dict[str, str]], temperature: float, max_tokens: int) -> str:\n pass\n\nclass OpenAIProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"gpt-4o\", base_url=None):\n super().__init__(api_key, model_name)\n if not OpenAI: raise ImportError(\"OpenAI library not installed. Run: pip install openai\")\n self.client = OpenAI(api_key=api_key, base_url=base_url)\n\n def generate(self, messages, temperature, max_tokens):\n response = self.client.chat.completions.create(\n model=self.model_name,\n messages=messages,\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.choices[0].message.content\n\nclass AnthropicProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"claude-sonnet-4-20250514\"):\n super().__init__(api_key, model_name)\n if not Anthropic: raise ImportError(\"Anthropic library not installed. Run: pip install anthropic\")\n self.client = Anthropic(api_key=api_key)\n\n def generate(self, messages, temperature, max_tokens):\n system_content = \"\"\n user_messages = []\n \n for msg in messages:\n if msg['role'] == 'system':\n system_content = msg['content']\n else:\n user_messages.append(msg)\n \n response = self.client.messages.create(\n model=self.model_name,\n system=system_content,\n messages=user_messages,\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.content[0].text\n\nclass CohereProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"command-a-03-2025\"):\n super().__init__(api_key, model_name)\n if not cohere: raise ImportError(\"Cohere library not installed. Run: pip install cohere\")\n self.client = cohere.Client(api_key=api_key)\n\n def generate(self, messages, temperature, max_tokens):\n chat_history = []\n message_content = \"\"\n \n for msg in messages:\n if msg['role'] == 'user':\n if not message_content: message_content = msg['content']\n elif msg['role'] == 'assistant':\n chat_history.append({\"role\": \"CHATBOT\", \"message\": msg['content']})\n elif msg['role'] == 'system':\n message_content = msg['content'] + \"\\n\\n\" + message_content\n\n response = self.client.chat(\n message=message_content,\n chat_history=chat_history,\n temperature=temperature,\n max_tokens=max_tokens,\n model=self.model_name\n )\n return response.text\n\nclass DeepSeekProvider(BaseLLMProvider):\n \"\"\"DeepSeek is largely OpenAI compatible\"\"\"\n def __init__(self, api_key, model_name=\"deepseek-chat\"):\n super().__init__(api_key, model_name)\n if not OpenAI: raise ImportError(\"OpenAI library required for DeepSeek. Run: pip install openai\")\n self.client = OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com\")\n\n def generate(self, messages, temperature, max_tokens):\n response = self.client.chat.completions.create(\n model=self.model_name,\n messages=messages,\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.choices[0].message.content\n\nclass GLMProvider(BaseLLMProvider):\n \"\"\"Z.ai (GLM) provider using existing ZaiClient logic\"\"\"\n def __init__(self, api_key, model_name=\"GLM-4.6V-Flash\"):\n super().__init__(api_key, model_name)\n if not ZaiClient: raise ImportError(\"Zai library not found.\")\n self.client = ZaiClient(api_key=api_key)\n\n def generate(self, messages, temperature, max_tokens):\n prompt = \"\"\n for msg in messages:\n role = msg['role'].upper()\n content = msg['content']\n prompt += f\"[{role}]: {content}\\n\"\n \n response = self.client.chat.completions.create(\n model=self.model_name,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.choices[0].message.content\n\nclass OllamaProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"llama3\", base_url=\"http://localhost:11434\"):\n super().__init__(api_key, model_name, base_url=base_url)\n if not ollama: raise ImportError(\"Ollama library not installed. Run: pip install ollama\")\n self.client = ollama.Client(host=base_url)\n\n def generate(self, messages, temperature, max_tokens):\n response = self.client.chat(\n model=self.model_name,\n messages=messages,\n options={\n 'temperature': temperature,\n 'num_predict': max_tokens\n }\n )\n return response['message']['content']\n\ndef get_provider_instance(provider_name: str, api_key: str, model_name: str, **kwargs):\n \"\"\"Factory function to get the provider instance.\"\"\"\n provider_map = {\n \"OpenAI\": OpenAIProvider,\n \"Anthropic\": AnthropicProvider,\n \"Cohere\": CohereProvider,\n \"DeepSeek\": DeepSeekProvider,\n \"GLM (Z.ai)\": GLMProvider,\n \"Ollama (Local)\": OllamaProvider\n }\n \n provider_class = provider_map.get(provider_name)\n if not provider_class:\n raise ValueError(f\"Provider {provider_name} not supported.\")\n \n return provider_class(api_key=api_key, model_name=model_name, **kwargs)\n\ndef query_llm(prompt, provider_name, api_key, model_name, temperature=0.1, max_tokens=8192):\n \"\"\"\n Generic query function replacing query_zai.\n Handles retries and delegates to specific providers.\n \"\"\"\n if not api_key and provider_name != \"Ollama (Local)\":\n return None\n \n extra_args = {}\n if provider_name == \"Ollama (Local)\":\n extra_args['base_url'] = st.session_state.get('ollama_base_url', 'http://localhost:11434')\n\n try:\n update_terminal_log(f\"Initializing {provider_name} Client...\", \"DEBUG\")\n except: pass\n\n max_retries = 10 \n provider = None\n\n for attempt in range(max_retries):\n try:\n try:\n update_terminal_log(f\"LLM Call Attempt {attempt + 1}/{max_retries} via {provider_name}...\", \"DEBUG\")\n except: pass\n \n \n provider = get_provider_instance(provider_name, api_key, model_name, **extra_args)\n \n messages = [{\"role\": \"user\", \"content\": prompt}]\n \n result_content = provider.generate(messages, temperature, max_tokens)\n \n del provider \n try: update_terminal_log(\"Response received successfully.\", \"SUCCESS\")\n except: pass\n return result_content\n \n except Exception as e:\n error_str = str(e)\n is_rate_limit = False\n \n \n rate_keywords = [\"429\", \"rate limit\", \"too many requests\", \"quota\", \"overload\", \"rate_limit_exceeded\"]\n if any(k in error_str.lower() for k in rate_keywords):\n is_rate_limit = True\n \n if is_rate_limit:\n try: update_terminal_log(f\"Rate Limit / Quota Exceeded detected.\", \"WARN\")\n except: pass\n \n wait_time = 15 * (2 ** attempt)\n if attempt < max_retries - 1:\n try: update_terminal_log(f\"Rate limit hit. Waiting {wait_time} seconds before retry...\", \"WARN\")\n except: pass\n time.sleep(wait_time)\n try: update_terminal_log(f\"Resuming retry...\", \"INFO\")\n except: pass\n else:\n try: update_terminal_log(\"Max retries reached for rate limit.\", \"ERROR\")\n except: pass\n return None\n else:\n try: update_terminal_log(f\"API Error: {error_str}\", \"ERROR\")\n except: pass\n if attempt < 3: time.sleep(2)\n else: return None\n \n return None"}, "38568b54d22416ceb9e6d6d23151f23ae29691ca": {"t": 1785846843.305339, "n": 33646, "h": "8f9300f0daddf937", "c": "import streamlit as st\nimport fitz \nimport time\nimport os \nimport re\nimport json\nimport json5\nimport pandas as pd\nimport io\nimport html\nimport hashlib\nimport gc\nfrom abc import ABC, abstractmethod\nfrom typing import List, Dict, Any\nfrom datetime import datetime\nfrom dotenv import load_dotenv\nimport firebase_admin\nfrom firebase_admin import credentials, firestore\nfrom cryptography.fernet import Fernet\nfrom fpdf import FPDF\nfrom streamlit_lottie import st_lottie\nimport streamlit.components.v1 as components\nimport base64\nimport pytesseract\nfrom PIL import Image\n\ntry:\n from openai import OpenAI\nexcept ImportError:\n OpenAI = None\n\ntry:\n from anthropic import Anthropic\nexcept ImportError:\n Anthropic = None\n\ntry:\n import cohere\nexcept ImportError:\n cohere = None\n\ntry:\n import ollama\nexcept ImportError:\n ollama = None\n\ntry:\n from zai import ZaiClient\nexcept ImportError:\n ZaiClient = None\n\nload_dotenv()\n\n\ndb = None \n\ntry:\n if not firebase_admin._apps:\n firebase_key_str = os.getenv(\"FIREBASE_KEY\")\n if firebase_key_str:\n cred_dict = json.loads(firebase_key_str)\n cred = credentials.Certificate(cred_dict)\n firebase_admin.initialize_app(cred)\n \n if firebase_admin._apps:\n db = firestore.client()\nexcept Exception as e:\n \n try:\n st.warning(f\"Error initializing Firebase: {e}\")\n except:\n print(f\"Error initializing Firebase: {e}\")\n\nMAX_LOG_ENTRIES = 200\nMAX_INPUT_TOKENS_SCREENER = 128000 \nMAX_INPUT_TOKENS_EXTRACTOR = 128000 \nMAX_OUTPUT_TOKENS = 8192\n\n\ndef get_firebase_stats():\n if db is None:\n return {\"papers_screened\": 0, \"papers_extracted\": 0}\n try:\n doc_ref = db.collection(\"ReviewAidAnalytics\").document(\"counters\")\n doc = doc_ref.get()\n \n if doc:\n return {\n \"papers_screened\": doc.to_dict().get(\"papers_screened\", 0),\n \"papers_extracted\": doc.to_dict().get(\"papers_extracted\", 0),\n \n }\n else:\n \n return {\"papers_screened\": 0, \"papers_extracted\": 0}\n except Exception:\n \n return {\"papers_screened\": 0, \"papers_extracted\": 0}\n\ndef display_citation_section():\n \"\"\"\n Displays the citation section.\n Checks session state to prevent duplicate element IDs on re-runs.\n \"\"\"\n\n\n st.markdown(\"---\")\n st.markdown(\"## Citation\")\n st.markdown(\"You can check ReviewAid's journal paper here: [Journal of Open Research Software](https://doi.org/10.5334/jors.672). If you are to use ReviewAid, please don't forget to cite the tool:\")\n\n apa_citation = (\n \"Sahu, V., & Balakrishnan, M. (2026). ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews. \"\n \"Journal of Open Research Software, 14(1), 21. https://doi.org/10.5334/jors.672\"\n )\n\n harvard_citation = (\n \"Sahu, V. and Balakrishnan, M., 2026. ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews. \"\n \"Journal of Open Research Software, 14(1), p. 21. Available at: https://doi.org/10.5334/jors.672\"\n )\n\n mla_citation = (\n \"Sahu, Vihaan, and Mohith Balakrishnan. \\\"ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews.\\\" \"\n \"Journal of Open Research Software, vol. 14, no. 1, 2026, p. 21.\"\n )\n\n chicago_citation = (\n \"Sahu, Vihaan, and Mohith Balakrishnan. 2026. \\\"ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews.\\\" \"\n \"Journal of Open Research Software 14 (1): 21. https://doi.org/10.5334/jors.672.\"\n )\n\n ieee_citation = (\n \"V. Sahu and M. Balakrishnan, \\\"ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews,\\\" \"\n \"Journal of Open Research Software, vol. 14, no. 1, p. 21, 2026. doi: 10.5334/jors.672.\"\n )\n\n vancouver_citation = (\n \"Sahu V, Balakrishnan M. ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews. \"\n \"J Open Res Softw. 2026;14(1):21. doi:10.5334/jors.672\"\n )\n\n ris_data = \"\"\"TY - JOUR\nAU - Sahu, V.\nAU - Balakrishnan, M.\nTI - ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews\nJO - Journal of Open Research Software\nPY - 2026\nVL - 14\nIS - 1\nSP - 21\nDO - 10.5334/jors.672\nER -\"\"\"\n\n bib_data = \"\"\"@article{Sahu2026,\n author = {Sahu, V. and Balakrishnan, M.},\n title = {ReviewAid: An Open-Source Tool for Efficient PICO-Based Screening and Data Extraction in Systematic Reviews},\n journal = {Journal of Open Research Software},\n year = {2026},\n volume = {14},\n number = {1},\n pages = {21},\n doi = {10.5334/jors.672}\n}\"\"\"\n \n try:\n citation_style = st.selectbox(\n \"Select citation style\",\n [\"APA\", \"Harvard\", \"MLA\", \"Chicago\", \"IEEE\", \"Vancouver\"]\n )\n except st.errors.StreamlitDuplicateElementId as e:\n st.warning(f\"Citation section skipped (Duplicate ID): {e}\")\n \n return\n\n if citation_style == \"APA\":\n citation_text = apa_citation\n elif citation_style == \"Harvard\":\n citation_text = harvard_citation\n elif citation_style == \"MLA\":\n citation_text = mla_citation\n elif citation_style == \"Chicago\":\n citation_text = chicago_citation\n elif citation_style == \"IEEE\":\n citation_text = ieee_citation\n elif citation_style == \"Vancouver\":\n citation_text = vancouver_citation\n\n escaped_citation = html.escape(citation_text)\n \n st.markdown(f'

{escaped_citation}

', unsafe_allow_html=True)\n\n js_citation_text = json.dumps(citation_text)\n \n st.markdown(f\"\"\"\n
\n \n RIS Format\n BibTeX Format\n
\n \"\"\", unsafe_allow_html=True)\n\n st.markdown(f\"\"\"\n \n \"\"\", unsafe_allow_html=True)\n\n\n\ndef init_analytics():\n pass\n\n\ndef increment_firebase_counter(field):\n if db is None: return\n try:\n doc_ref = db.collection(\"ReviewAidAnalytics\").document(\"counters\")\n doc_ref.update({field: firestore.Increment(1)})\n except Exception as e: print(f\"Analytics Error: {e}\")\n\ndef update_processing_stats(mode, count=1):\n if db is None: return\n if mode == \"screener\":\n for _ in range(count): increment_firebase_counter(\"papers_screened\")\n elif mode == \"extractor\":\n for _ in range(count): increment_firebase_counter(\"papers_extracted\")\n\ndef update_terminal_log(msg, level=\"INFO\"):\n if \"terminal_logs\" not in st.session_state or \"terminal_placeholder\" not in st.session_state:\n return\n\n timestamp = datetime.now().strftime(\"%H:%M:%S.%f\")[:-3] \n \n if level == \"INFO\":\n color_class = \"log-info\"\n elif level == \"SUCCESS\":\n color_class = \"log-success\"\n elif level == \"WARN\":\n color_class = \"log-warn\"\n elif level == \"ERROR\":\n color_class = \"log-error\"\n elif level == \"SYSTEM\":\n color_class = \"log-system\"\n elif level == \"DEBUG\":\n color_class = \"log-debug\"\n else:\n color_class = \"log-info\"\n \n log_entry = f'[{timestamp}]{level}: {html.escape(msg)}'\n \n st.session_state.terminal_logs.append(log_entry)\n \n if len(st.session_state.terminal_logs) > MAX_LOG_ENTRIES:\n st.session_state.terminal_logs.pop(0)\n \n current_time = time.time()\n if current_time - st.session_state.last_log_update_time > 0.5:\n full_log_html = '
' + \"\".join(st.session_state.terminal_logs) + '
'\n scroll_script = \"\"\"\n \n \"\"\"\n try:\n st.session_state.terminal_placeholder.markdown(full_log_html + scroll_script, unsafe_allow_html=True)\n except Exception:\n pass\n st.session_state.last_log_update_time = current_time\n\ndef extract_pdf_content(pdf_bytes, enable_ocr=False):\n try:\n update_terminal_log(\"Initializing PDF extraction engine (PyMuPDF)...\", \"DEBUG\")\n except:\n pass\n \n doc = None\n try:\n doc = fitz.open(stream=pdf_bytes, filetype=\"pdf\")\n page_count = doc.page_count\n try:\n update_terminal_log(f\"Document opened successfully. Total pages: {page_count}\", \"INFO\")\n except:\n pass\n \n MAX_CHAR_LIMIT = 600000 \n full_text_parts = []\n current_length = 0\n references_found = False\n \n \n ocr_text_parts = []\n total_images_found = 0\n \n for i, page in enumerate(doc):\n if current_length > MAX_CHAR_LIMIT and not references_found:\n try:\n update_terminal_log(f\"Token limit reached at Page {i+1}. Stopping PDF read.\", \"INFO\")\n except:\n pass\n break\n \n page_text = page.get_text()\n \n if re.search(r'(?:\\n|\\r\\n){1,2}(References|Reference|Bibliography)(?:\\s|\\r?\\n|$)', page_text, re.IGNORECASE):\n references_found = True \n\n full_text_parts.append(page_text)\n current_length += len(page_text)\n \n \n if enable_ocr:\n try:\n image_list = page.get_images(full=True)\n if image_list:\n num_images_on_page = len(image_list)\n update_terminal_log(f\"Page {i+1}: Found {num_images_on_page} image(s). Running Tesseract OCR...\", \"INFO\")\n \n page_ocr_success = False\n \n for img in image_list:\n try:\n xref = img[0]\n base_image = doc.extract_image(xref)\n image_bytes = base_image[\"image\"]\n image = Image.open(io.BytesIO(image_bytes))\n \n \n extracted_text = pytesseract.image_to_string(image)\n \n if extracted_text.strip():\n ocr_text_parts.append(extracted_text)\n page_ocr_success = True\n \n \n del image, image_bytes\n \n except Exception:\n pass \n \n if page_ocr_success:\n update_terminal_log(f\"Page {i+1}: OCR complete.\", \"SUCCESS\")\n \n total_images_found += num_images_on_page\n except Exception as e:\n update_terminal_log(f\"Page {i+1}: Error scanning for images - {str(e)}\", \"ERROR\")\n \n \n del page_text \n \n full_text = \"\".join(full_text_parts)\n del full_text_parts\n \n\n \n if enable_ocr:\n if total_images_found > 0:\n ocr_content = \"\\n\\n\".join(ocr_text_parts)\n full_text += \"\\n\\n[OCR EXTRACTED CONTENT]:\\n\" + ocr_content\n update_terminal_log(f\"Finished. Total images OCR'd: {total_images_found} (Data Extracted from Images)\", \"SUCCESS\")\n else:\n update_terminal_log(\"OCR Enabled, but no images were found in this document.\", \"WARN\")\n\n\n ref_match = re.search(r'(?:\\n|\\r\\n){1,2}(References|Reference|Bibliography)(?:\\s|\\r?\\n|$)', full_text, re.IGNORECASE)\n \n if ref_match:\n main_text_end = ref_match.start()\n full_text = full_text[:main_text_end]\n try:\n update_terminal_log(\"References section detected and removed to save tokens.\", \"INFO\")\n except:\n pass\n del ref_match\n\n metadata = doc.metadata\n title = metadata.get('title', '')\n author = metadata.get('author', '')\n del metadata \n \n try:\n update_terminal_log(f\"Metadata read -> Title: '{title}', Author: '{author}'\", \"DEBUG\")\n except:\n pass\n \n if not title or len(title) < 5 or \"microsoft\" in title.lower() or \".doc\" in title.lower() or title.isdigit():\n try:\n update_terminal_log(\"Metadata title appears empty or invalid. Scanning first page text for title...\", \"DEBUG\")\n except:\n pass\n \n first_page_text = full_text[:3000]\n lines = first_page_text.split('\\n')\n \n for line in lines:\n clean_line = line.strip()\n\n if len(clean_line) > 10 and len(clean_line) < 200:\n if not re.match(r'^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}$', clean_line, re.IGNORECASE) and \\\n not clean_line.startswith(\"http\") and \\\n \"abstract\" not in clean_line.lower() and \\\n \"introduction\" not in clean_line.lower() and \\\n \"keywords\" not in clean_line.lower() and \\\n \"page\" not in clean_line.lower() and \\\n \"vol.\" not in clean_line.lower():\n \n\n title = clean_line\n try:\n update_terminal_log(f\"Found candidate title in text: '{title[:50]}...'\", \"INFO\")\n except:\n pass\n break\n \n year = ''\n if not year:\n if page_count > 0:\n year_match = re.search(r'\\b(19|20)\\d{2}\\b', full_text[:5000]) \n if year_match:\n year = year_match.group()\n try:\n update_terminal_log(f\"Year found on Page 1: {year}\", \"DEBUG\")\n except:\n pass\n \n if not year:\n creation_date = doc.metadata.get('creationDate', '')\n if creation_date:\n year_match = re.search(r'\\b(19|20)\\d{2}\\b', creation_date)\n if year_match:\n year = year_match.group()\n try:\n update_terminal_log(f\"Year derived from creationDate: {year}\", \"DEBUG\")\n except:\n pass\n \n # Perform garbage collection if OCR was used to make space to save RAM\n if enable_ocr and total_images_found > 0:\n gc.collect()\n update_terminal_log(\"OCR'd images will be cleared to save space.\", \"SYSTEM\")\n\n return full_text, title, author, year\n \n except Exception as e:\n try:\n update_terminal_log(f\"Error during PDF extraction: {str(e)}\", \"ERROR\")\n except:\n pass\n return \"\", \"\", \"\", \"\"\n finally:\n if doc:\n doc.close()\n del doc\n\ndef preprocess_text_for_ai(text, max_tokens=MAX_INPUT_TOKENS_SCREENER):\n if \" \" in text or \"\\n\" in text:\n text = \" \".join(text.split())\n \n char_limit = max_tokens * 4\n if len(text) > char_limit:\n try:\n update_terminal_log(f\"Text exceeds token limit ({len(text)} > {char_limit}). Truncating...\", \"WARN\")\n except:\n pass\n text = text[:char_limit]\n \n return text\n\ndef to_docx(df):\n from docx import Document\n from docx.shared import Inches, Pt, RGBColor\n from docx.enum.text import WD_ALIGN_PARAGRAPH\n from docx.oxml.ns import qn\n from docx.oxml import OxmlElement\n\n doc = Document()\n \n sections = doc.sections\n for section in sections:\n section.top_margin = Inches(0.5)\n section.bottom_margin = Inches(0.5) \n section.left_margin = Inches(0.5)\n section.right_margin = Inches(0.5)\n\n section = doc.sections[0]\n \n section.page_width = Inches(11.69)\n section.page_height = Inches(8.27)\n\n header = section.header\n header_para = header.paragraphs[0]\n header_para.alignment = WD_ALIGN_PARAGRAPH.RIGHT\n \n logo_path = os.path.join(\"assets\", \"RA_transparent.png\")\n if os.path.exists(logo_path):\n try:\n run_logo = header_para.add_run()\n run_logo.add_picture(logo_path, height=Inches(0.4))\n except Exception:\n pass \n\n footer = section.footer\n footer_para = footer.paragraphs[0]\n footer_para.alignment = WD_ALIGN_PARAGRAPH.LEFT\n \n generated_date = datetime.now().strftime(\"%B %d, %Y\")\n \n footer_text = f\"Generated by ReviewAid on {generated_date}\"\n \n footer_run = footer_para.add_run(footer_text)\n footer_run.font.name = 'Arial'\n footer_run.font.size = Pt(8)\n footer_run.font.color.rgb = RGBColor(100, 100, 100)\n\n title = doc.add_heading('Results', 0)\n title.alignment = WD_ALIGN_PARAGRAPH.CENTER\n \n num_cols = len(df.columns)\n table = doc.add_table(rows=1, cols=num_cols)\n\n table.style = 'Table Grid' \n \n table.allow_autofit = True\n for row in table.rows:\n row.allow_break_across_pages = True\n\n usable_width = section.page_width - section.left_margin - section.right_margin - Inches(0.2)\n col_width = usable_width / num_cols if num_cols > 0 else usable_width\n\n hdr_cells = table.rows[0].cells\n for i, col in enumerate(df.columns):\n cell = hdr_cells[i]\n cell.text = str(col)\n cell.width = col_width \n \n cell_par = cell.paragraphs[0]\n cell_par.runs[0].font.bold = True\n cell_par.runs[0].font.name = 'Arial'\n cell_par.runs[0].font.size = Pt(10)\n cell_par.runs[0].font.color.rgb = RGBColor(0, 0, 0) \n cell_par.alignment = WD_ALIGN_PARAGRAPH.CENTER\n \n shading_elm = OxmlElement('w:shd')\n shading_elm.set(qn('w:fill'), 'FFFFFF') \n cell._element.get_or_add_tcPr().append(shading_elm)\n \n tc_borders = OxmlElement('w:tcBorders')\n bottom = OxmlElement('w:bottom')\n bottom.set(qn('w:val'), 'single')\n bottom.set(qn('w:sz'), '4')\n bottom.set(qn('w:space'), '0')\n bottom.set(qn('w:color'), '000000') \n tc_borders.append(bottom)\n cell._element.get_or_add_tcPr().append(tc_borders)\n\n for _, row in df.iterrows():\n row_cells = table.add_row().cells\n for i, val in enumerate(row):\n cell = row_cells[i]\n cell.width = col_width \n \n para = cell.paragraphs[0]\n \n safe_val = str(val)\n safe_val = re.sub(r'[\\x00-\\x08\\x0b-\\x0c\\x0e-\\x1f\\x7f-\\x9f]', '', safe_val)\n \n run = para.add_run(safe_val)\n run.font.name = 'Arial'\n run.font.size = Pt(9)\n run.font.color.rgb = RGBColor(0, 0, 0) \n \n tc = cell._tc\n tcPr = tc.get_or_add_tcPr()\n shading_elm = OxmlElement('w:shd')\n shading_elm.set(qn('w:fill'), 'FFFFFF')\n tcPr.append(shading_elm)\n\n buffer = io.BytesIO()\n doc.save(buffer)\n buffer.seek(0)\n return buffer.getvalue()\n\ndef to_pdf(df):\n pdf = FPDF(orientation='L', unit='mm', format='A4')\n pdf.add_page()\n \n pdf.set_auto_page_break(auto=True, margin=15) \n \n pdf.set_font(\"Arial\", size=8)\n\n logo_path = os.path.join(\"assets\", \"RA_transparent.png\")\n if os.path.exists(logo_path):\n try:\n pdf.image(logo_path, x=250, y=10, w=30)\n except:\n pass \n\n generated_date = datetime.now().strftime(\"%Y-%m-%d\")\n\n col_width = 277 / len(df.columns) \n line_height = 5 \n\n pdf.set_fill_color(65, 137, 220) \n pdf.set_text_color(255, 255, 255)\n pdf.set_font(\"Arial\", style='B', size=8)\n \n if pdf.get_y() > 260: \n pdf.add_page()\n \n for col in df.columns:\n pdf.cell(col_width, 7, str(col), border=1, align='C', fill=True)\n pdf.ln(7)\n \n pdf.set_text_color(0, 0, 0)\n pdf.set_font(\"Arial\", size=7)\n is_odd = True\n \n table_x_start = pdf.get_x()\n \n for index, row in df.iterrows():\n\n if pdf.get_y() > 250:\n pdf.add_page()\n\n pdf.set_y(10)\n pdf.set_fill_color(65, 137, 220) \n pdf.set_text_color(255, 255, 255)\n pdf.set_font(\"Arial\", style='B', size=8)\n for col in df.columns:\n pdf.cell(col_width, 7, str(col), border=1, align='C', fill=True)\n pdf.ln(7)\n \n pdf.set_text_color(0, 0, 0)\n pdf.set_font(\"Arial\", size=7)\n is_odd = True\n table_x_start = pdf.get_x()\n\n if not is_odd:\n pdf.set_fill_color(245, 245, 245)\n else:\n pdf.set_fill_color(255, 255, 255)\n\n y_start = pdf.get_y()\n max_y_end = y_start\n \n col_heights = []\n for col_name, val in zip(df.columns, row):\n clean_text = str(val).encode('latin-1', 'replace').decode('latin-1')\n \n chars_per_line = int(col_width * 3.5) \n lines = len(clean_text) // chars_per_line + 1\n \n lines += clean_text.count('\\n')\n \n cell_h = lines * line_height\n col_heights.append(cell_h)\n if cell_h > (max_y_end - y_start):\n max_y_end = y_start + cell_h\n \n\n row_height = max_y_end - y_start\n if row_height < line_height:\n row_height = line_height\n\n x_cursor = table_x_start\n \n for i, val in enumerate(row):\n clean_text = str(val).encode('latin-1', 'replace').decode('latin-1')\n\n pdf.set_xy(x_cursor, y_start)\n \n pdf.multi_cell(col_width, row_height, clean_text, border=1, fill=(not is_odd), align='L')\n \n x_cursor += col_width\n\n pdf.set_y(max_y_end)\n pdf.set_x(table_x_start) \n \n is_odd = not is_odd\n\n pdf.set_y(-15)\n pdf.set_font(\"Arial\", size=7, style='I')\n pdf.set_text_color(128, 128, 128)\n pdf.set_x(10) \n pdf.cell(0, 10, f\"Generated by ReviewAid on {generated_date}\", 0, 0, 'L')\n \n return pdf.output(dest='S').encode('latin1')\n\ndef to_csv(df):\n buffer = io.StringIO()\n buffer.write(\"# Generated by ReviewAid\\n\")\n df.to_csv(buffer, index=False)\n return buffer.getvalue().encode('utf-8')\n\ndef to_excel(df):\n import xlsxwriter\n buffer = io.BytesIO()\n with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer:\n df.to_excel(writer, index=False, sheet_name='ReviewAid Data')\n \n workbook = writer.book\n worksheet = writer.sheets['ReviewAid Data']\n \n header_format = workbook.add_format({\n 'bold': True,\n 'text_wrap': True,\n 'valign': 'top',\n 'fg_color': '#4189DC', \n 'font_color': '#FFFFFF',\n 'border': 1\n })\n \n for col_num, value in enumerate(df.columns.values):\n worksheet.write(0, col_num, value, header_format)\n \n generated_date = datetime.now().strftime(\"%Y-%m-%d\")\n\n worksheet.set_footer(f'&LGenerated by ReviewAid on {generated_date}')\n \n buffer.seek(0)\n return buffer.getvalue()\n\n\n\nclass BaseLLMProvider(ABC):\n \"\"\"Abstract base class for LLM providers.\"\"\"\n def __init__(self, api_key: str, model_name: str, **kwargs):\n self.api_key = api_key\n self.model_name = model_name\n self.extra_params = kwargs\n\n @abstractmethod\n def generate(self, messages: List[Dict[str, str]], temperature: float, max_tokens: int) -> str:\n pass\n\nclass OpenAIProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"gpt-4o\", base_url=None):\n super().__init__(api_key, model_name)\n if not OpenAI: raise ImportError(\"OpenAI library not installed. Run: pip install openai\")\n self.client = OpenAI(api_key=api_key, base_url=base_url)\n\n def generate(self, messages, temperature, max_tokens):\n response = self.client.chat.completions.create(\n model=self.model_name,\n messages=messages,\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.choices[0].message.content\n\nclass AnthropicProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"claude-sonnet-4-20250514\"):\n super().__init__(api_key, model_name)\n if not Anthropic: raise ImportError(\"Anthropic library not installed. Run: pip install anthropic\")\n self.client = Anthropic(api_key=api_key)\n\n def generate(self, messages, temperature, max_tokens):\n system_content = \"\"\n user_messages = []\n \n for msg in messages:\n if msg['role'] == 'system':\n system_content = msg['content']\n else:\n user_messages.append(msg)\n \n response = self.client.messages.create(\n model=self.model_name,\n system=system_content,\n messages=user_messages,\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.content[0].text\n\nclass CohereProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"command-a-03-2025\"):\n super().__init__(api_key, model_name)\n if not cohere: raise ImportError(\"Cohere library not installed. Run: pip install cohere\")\n self.client = cohere.Client(api_key=api_key)\n\n def generate(self, messages, temperature, max_tokens):\n chat_history = []\n message_content = \"\"\n \n for msg in messages:\n if msg['role'] == 'user':\n if not message_content: message_content = msg['content']\n elif msg['role'] == 'assistant':\n chat_history.append({\"role\": \"CHATBOT\", \"message\": msg['content']})\n elif msg['role'] == 'system':\n message_content = msg['content'] + \"\\n\\n\" + message_content\n\n response = self.client.chat(\n message=message_content,\n chat_history=chat_history,\n temperature=temperature,\n max_tokens=max_tokens,\n model=self.model_name\n )\n return response.text\n\nclass DeepSeekProvider(BaseLLMProvider):\n \"\"\"DeepSeek is largely OpenAI compatible\"\"\"\n def __init__(self, api_key, model_name=\"deepseek-chat\"):\n super().__init__(api_key, model_name)\n if not OpenAI: raise ImportError(\"OpenAI library required for DeepSeek. Run: pip install openai\")\n self.client = OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com\")\n\n def generate(self, messages, temperature, max_tokens):\n response = self.client.chat.completions.create(\n model=self.model_name,\n messages=messages,\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.choices[0].message.content\n\nclass GLMProvider(BaseLLMProvider):\n \"\"\"Z.ai (GLM) provider using existing ZaiClient logic\"\"\"\n def __init__(self, api_key, model_name=\"GLM-4.6V-Flash\"):\n super().__init__(api_key, model_name)\n if not ZaiClient: raise ImportError(\"Zai library not found.\")\n self.client = ZaiClient(api_key=api_key)\n\n def generate(self, messages, temperature, max_tokens):\n prompt = \"\"\n for msg in messages:\n role = msg['role'].upper()\n content = msg['content']\n prompt += f\"[{role}]: {content}\\n\"\n \n response = self.client.chat.completions.create(\n model=self.model_name,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=temperature,\n max_tokens=max_tokens\n )\n return response.choices[0].message.content\n\nclass OllamaProvider(BaseLLMProvider):\n def __init__(self, api_key, model_name=\"llama3\", base_url=\"http://localhost:11434\"):\n super().__init__(api_key, model_name, base_url=base_url)\n if not ollama: raise ImportError(\"Ollama library not installed. Run: pip install ollama\")\n self.client = ollama.Client(host=base_url)\n\n def generate(self, messages, temperature, max_tokens):\n response = self.client.chat(\n model=self.model_name,\n messages=messages,\n options={\n 'temperature': temperature,\n 'num_predict': max_tokens\n }\n )\n return response['message']['content']\n\ndef get_provider_instance(provider_name: str, api_key: str, model_name: str, **kwargs):\n \"\"\"Factory function to get the provider instance.\"\"\"\n provider_map = {\n \"OpenAI\": OpenAIProvider,\n \"Anthropic\": AnthropicProvider,\n \"Cohere\": CohereProvider,\n \"DeepSeek\": DeepSeekProvider,\n \"GLM (Z.ai)\": GLMProvider,\n \"Ollama (Local)\": OllamaProvider\n }\n \n provider_class = provider_map.get(provider_name)\n if not provider_class:\n raise ValueError(f\"Provider {provider_name} not supported.\")\n \n return provider_class(api_key=api_key, model_name=model_name, **kwargs)\n\ndef query_llm(prompt, provider_name, api_key, model_name, temperature=0.1, max_tokens=8192):\n \"\"\"\n Generic query function replacing query_zai.\n Handles retries and delegates to specific providers.\n \"\"\"\n if not api_key and provider_name != \"Ollama (Local)\":\n return None\n \n extra_args = {}\n if provider_name == \"Ollama (Local)\":\n extra_args['base_url'] = st.session_state.get('ollama_base_url', 'http://localhost:11434')\n\n try:\n update_terminal_log(f\"Initializing {provider_name} Client...\", \"DEBUG\")\n except: pass\n\n max_retries = 10 \n provider = None\n\n for attempt in range(max_retries):\n try:\n try:\n update_terminal_log(f\"LLM Call Attempt {attempt + 1}/{max_retries} via {provider_name}...\", \"DEBUG\")\n except: pass\n \n \n provider = get_provider_instance(provider_name, api_key, model_name, **extra_args)\n \n messages = [{\"role\": \"user\", \"content\": prompt}]\n \n result_content = provider.generate(messages, temperature, max_tokens)\n \n del provider \n try: update_terminal_log(\"Response received successfully.\", \"SUCCESS\")\n except: pass\n return result_content\n \n except Exception as e:\n error_str = str(e)\n is_rate_limit = False\n \n \n rate_keywords = [\"429\", \"rate limit\", \"too many requests\", \"quota\", \"overload\", \"rate_limit_exceeded\"]\n if any(k in error_str.lower() for k in rate_keywords):\n is_rate_limit = True\n \n if is_rate_limit:\n try: update_terminal_log(f\"Rate Limit / Quota Exceeded detected.\", \"WARN\")\n except: pass\n \n wait_time = 15 * (2 ** attempt)\n if attempt < max_retries - 1:\n try: update_terminal_log(f\"Rate limit hit. Waiting {wait_time} seconds before retry...\", \"WARN\")\n except: pass\n time.sleep(wait_time)\n try: update_terminal_log(f\"Resuming retry...\", \"INFO\")\n except: pass\n else:\n try: update_terminal_log(\"Max retries reached for rate limit.\", \"ERROR\")\n except: pass\n return None\n else:\n try: update_terminal_log(f\"API Error: {error_str}\", \"ERROR\")\n except: pass\n if attempt < 3: time.sleep(2)\n else: return None\n \n return None"}}}}