{"repo": "abdullahwali007/nasa-biological-knowledge-engine", "files": {"nasa_engine.py": {"2c542eaf8af2682c48d36c36a5b2091c744bfdb6": {"t": 1785674300.4602125, "n": 34858, "h": "b529d05f41cb5953", "c": "#!/usr/bin/env python3\r\n\"\"\"\r\nEngine Streamlit App — Improved search & accurate dataset summaries with PMC publications\r\n\r\nSave as: engine_streamlit.py\r\nRun: streamlit run engine_streamlit.py\r\n\r\nRequirements:\r\n    pip install pandas scikit-learn streamlit openai\r\n\r\nPut index.json and SB_publication_PMC.json in same folder.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\nimport os\r\nimport re\r\nimport json\r\nimport time\r\nimport logging\r\nimport webbrowser\r\nfrom pathlib import Path\r\nfrom typing import Dict, Any, List, Optional, Tuple\r\nimport base64\r\n\r\nimport pandas as pd\r\nfrom sklearn.feature_extraction.text import TfidfVectorizer\r\nfrom sklearn.metrics.pairwise import cosine_similarity\r\nimport numpy as np\r\nimport streamlit as st\r\nfrom openai import OpenAI  \r\n\r\n# ---------------- CONFIG ----------------\r\nINDEX_PATH = Path(\"index.json\")\r\nPUBLICATIONS_PATH = Path(\"SB_publication_PMC.json\")\r\n\r\n# LongCat Configuration\r\nLONGCAT_MODEL = \"LongCat-Flash-Chat\" \r\nLONGCAT_BASE_URL = \"https://api.longcat.chat/openai/v1\"\r\n# We check secrets first, but fallback to your provided key\r\nDEFAULT_LONGCAT_KEY = 'ak_1ZZ4nn04h3GB7Hj2Zm2g36nx0mD2J' \r\n\r\nTOP_K_DATASETS = 5\r\nTOP_K_PUBLICATIONS = 8\r\nSIMILARITY_THRESHOLD = 0.02\r\nANALYSIS_ROWS = 1000        # how many rows to read for column stats (cap)\r\nPREVIEW_ROWS = 5            # rows shown in quick preview\r\nMAX_API_ATTEMPTS = 3\r\n# ----------------------------------------\r\n\r\nlogging.basicConfig(level=logging.INFO, format=\"%(levelname)s: %(message)s\")\r\nlogger = logging.getLogger(\"EngineStreamlit\")\r\n\r\nURL_REGEX = re.compile(r\"(https?://[^\\s'\\\"<>\\)\\]]+)\", re.IGNORECASE)\r\nNUMBER_REGEX = re.compile(r\"\\b(?:\\d{2,}|[0-9]+(?:\\.[0-9]+)?)\\b\")\r\n\r\n\r\n# ---------- Utilities ----------\r\ndef extract_urls(text: str) -> List[str]:\r\n    if not text:\r\n        return []\r\n    found = URL_REGEX.findall(text)\r\n    seen = set()\r\n    out = []\r\n    for u in found:\r\n        u = u.rstrip(\".,;:)\")\r\n        if u not in seen:\r\n            out.append(u)\r\n            seen.add(u)\r\n    return out\r\n\r\n\r\ndef safe_read_table(path: Path, nrows: int = PREVIEW_ROWS) -> Optional[pd.DataFrame]:\r\n    \"\"\"Try reading a table; return None if fails. Use small memory-friendly reads.\"\"\"\r\n    for sep in (\"\\t\", \",\"):\r\n        try:\r\n            return pd.read_csv(path, sep=sep, nrows=nrows, low_memory=False)\r\n        except Exception:\r\n            continue\r\n    return None\r\n\r\n\r\ndef read_table_for_analysis(path: Path, nrows: int = ANALYSIS_ROWS) -> Optional[pd.DataFrame]:\r\n    \"\"\"Read more rows for statistical analysis. Falls back gracefully.\"\"\"\r\n    for sep in (\"\\t\", \",\"):\r\n        try:\r\n            return pd.read_csv(path, sep=sep, nrows=nrows, low_memory=False)\r\n        except Exception:\r\n            continue\r\n    return None\r\n\r\n\r\ndef is_numeric_series(s: pd.Series) -> bool:\r\n    try:\r\n        return pd.api.types.is_numeric_dtype(s)\r\n    except Exception:\r\n        # fallback: attempt to coerce\r\n        try:\r\n            pd.to_numeric(s.dropna().iloc[:50])\r\n            return True\r\n        except Exception:\r\n            return False\r\n\r\n\r\ndef analyze_dataframe(df: pd.DataFrame, top_n: int = 6) -> Dict[str, Any]:\r\n    \"\"\"\r\n    Compute column-level summaries:\r\n      - For numeric columns: count, mean, median, std, min, max, unique_count, top frequencies\r\n      - For non-numeric: unique_count, top frequencies\r\n    Returns dict: {col_name: {summary...}}\r\n    \"\"\"\r\n    summaries = {}\r\n    for col in df.columns:\r\n        try:\r\n            series = df[col].dropna()\r\n            if series.empty:\r\n                summaries[col] = {\"note\": \"empty\", \"count\": 0}\r\n                continue\r\n            # try numeric detection\r\n            numeric = is_numeric_series(series)\r\n            info: Dict[str, Any] = {\"count\": int(len(series)), \"unique\": int(series.nunique())}\r\n            if numeric:\r\n                # coerce numeric\r\n                snum = pd.to_numeric(series, errors=\"coerce\").dropna()\r\n                if snum.empty:\r\n                    numeric = False\r\n                else:\r\n                    info.update({\r\n                        \"numeric\": True,\r\n                        \"mean\": float(snum.mean()),\r\n                        \"median\": float(snum.median()),\r\n                        \"std\": float(snum.std()) if len(snum) > 1 else 0.0,\r\n                        \"min\": float(snum.min()),\r\n                        \"max\": float(snum.max()),\r\n                    })\r\n                    # compute top frequencies\r\n                    vc = snum.value_counts().head(top_n)\r\n                    top = []\r\n                    total = len(snum)\r\n                    for val, cnt in vc.items():\r\n                        try:\r\n                            value = float(str(val))\r\n                        except (ValueError, TypeError):\r\n                            value = val\r\n                        top.append({\"value\": value, \"count\": int(cnt), \"pct\": round(100.0 * cnt / total, 2)})\r\n                    info[\"top\"] = top\r\n            if not numeric:\r\n                # treat as categorical/text\r\n                vc = series.astype(str).value_counts().head(top_n)\r\n                total = len(series)\r\n                top = []\r\n                for val, cnt in vc.items():\r\n                    top.append({\"value\": str(val), \"count\": int(cnt), \"pct\": round(100.0 * cnt / total, 2)})\r\n                info[\"top\"] = top\r\n            summaries[col] = info\r\n        except Exception as e:\r\n            summaries[col] = {\"error\": str(e)}\r\n    return summaries\r\n\r\n\r\n# ---------- Search index building ----------\r\nclass SearchIndex:\r\n    \"\"\"\r\n    Builds a TF-IDF index over datasets and publications.\r\n    \"\"\"\r\n    def __init__(self, data_index: Dict[str, Any], publications: List[Dict[str, str]]):\r\n        self.data_index = data_index or {}\r\n        self.publications = publications or []\r\n        self.keys: List[str] = []\r\n        self.corpus: List[str] = []\r\n        self.entry_types: List[str] = []  # \"dataset\" or \"publication\"\r\n        self.vectorizer: Optional[TfidfVectorizer] = None\r\n        self.matrix = None\r\n        self._build_index()\r\n\r\n    @staticmethod\r\n    def _make_dataset_text(ds: str, meta: Dict[str, Any]) -> Tuple[str, List[str]]:\r\n        parts = [f\"DATASET_ID: {ds}\"]\r\n        urls: List[str] = []\r\n        # include metadata fields\r\n        for k in (\"organism\", \"assay_types\", \"description\", \"folder\"):\r\n            v = meta.get(k)\r\n            if v:\r\n                if isinstance(v, list):\r\n                    parts.append(\" \".join(map(str, v)))\r\n                else:\r\n                    parts.append(str(v))\r\n                if isinstance(v, str):\r\n                    urls.extend(extract_urls(v))\r\n        # include column names & small textual preview\r\n        files = meta.get(\"files\", {})\r\n        file_count = 0\r\n        for name, entry in list(files.items())[:4]:\r\n            path = Path(entry if isinstance(entry, str) else (entry[0] if entry else \"\"))\r\n            if not path.exists():\r\n                continue\r\n            try:\r\n                df_preview = pd.read_csv(path, sep=None, nrows=50, engine=\"python\", low_memory=False)\r\n            except Exception:\r\n                try:\r\n                    df_preview = pd.read_csv(path, nrows=20, low_memory=False)\r\n                except Exception:\r\n                    # try read as text\r\n                    try:\r\n                        txt = path.read_text(encoding=\"utf-8\", errors=\"ignore\")[:2000]\r\n                        parts.append(txt)\r\n                        urls.extend(extract_urls(txt))\r\n                    except Exception:\r\n                        pass\r\n                else:\r\n                    pass\r\n            else:\r\n                # include column names & first few rows\r\n                colnames = \" \".join(map(str, df_preview.columns[:20]))\r\n                parts.append(colnames)\r\n                try:\r\n                    sample_text = \" \".join(df_preview.head(5).astype(str).agg(\" \".join, axis=1).tolist())\r\n                    parts.append(sample_text)\r\n                    urls.extend(extract_urls(sample_text))\r\n                except Exception:\r\n                    pass\r\n            file_count += 1\r\n            if file_count >= 4:\r\n                break\r\n\r\n        text = \"\\n\".join(parts)\r\n        return text, list(dict.fromkeys(urls))\r\n\r\n    def _build_index(self):\r\n        self.keys = []\r\n        self.corpus = []\r\n        self.entry_types = []\r\n        \r\n        # Index datasets\r\n        for ds, meta in self.data_index.items():\r\n            txt, urls = self._make_dataset_text(ds, meta)\r\n            self.keys.append(ds)\r\n            self.corpus.append(txt if txt else ds)\r\n            self.entry_types.append(\"dataset\")\r\n\r\n        # Index publications\r\n        for idx, pub in enumerate(self.publications):\r\n            title = pub.get(\"Title\", \"\")\r\n            link = pub.get(\"Link\", \"\")\r\n            pub_text = f\"PUBLICATION: {title} {link}\"\r\n            self.keys.append(f\"pub_{idx}\")\r\n            self.corpus.append(pub_text)\r\n            self.entry_types.append(\"publication\")\r\n\r\n        # vectorize\r\n        try:\r\n            self.vectorizer = TfidfVectorizer(stop_words=\"english\", ngram_range=(1, 2), max_features=10000)\r\n            self.matrix = self.vectorizer.fit_transform(self.corpus)\r\n            logger.info(\"Search index built: %d datasets, %d publications\", \r\n                       len([t for t in self.entry_types if t == \"dataset\"]),\r\n                       len([t for t in self.entry_types if t == \"publication\"]))\r\n        except Exception as e:\r\n            logger.warning(\"Failed to build TF-IDF index: %s\", e)\r\n            self.vectorizer = None\r\n            self.matrix = None\r\n\r\n    def rebuild(self, data_index: Dict[str, Any], publications: List[Dict[str, str]]):\r\n        self.data_index = data_index or {}\r\n        self.publications = publications or []\r\n        self._build_index()\r\n\r\n    def query(self, q: str, top_k_datasets: int = TOP_K_DATASETS, top_k_publications: int = TOP_K_PUBLICATIONS) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:\r\n        q_norm = q.strip()\r\n        q_tokens = set(re.findall(r\"\\w+\", q_norm.lower()))\r\n        numbers_in_q = set(NUMBER_REGEX.findall(q_norm))\r\n        scores = []\r\n        \r\n        if self.vectorizer and self.matrix is not None:\r\n            try:\r\n                qv = self.vectorizer.transform([q_norm])\r\n                sims = cosine_similarity(qv, self.matrix).flatten()\r\n            except Exception:\r\n                sims = np.zeros(len(self.keys))\r\n        else:\r\n            sims = np.zeros(len(self.keys))\r\n\r\n        for idx, key in enumerate(self.keys):\r\n            entry_type = self.entry_types[idx]\r\n            sim = float(sims[idx]) if idx < len(sims) else 0.0\r\n            corpus_tokens = set(re.findall(r\"\\w+\", (self.corpus[idx] or \"\").lower()))\r\n            overlap = len(q_tokens & corpus_tokens) / (1 + len(q_tokens))\r\n            score = 0.7 * sim + 0.3 * overlap\r\n            if numbers_in_q:\r\n                found_num_boost = 0.0\r\n                for num in numbers_in_q:\r\n                    if num in (self.corpus[idx] or \"\"):\r\n                        found_num_boost += 0.5\r\n                score += found_num_boost\r\n            scores.append((key, entry_type, float(score), float(sim), float(overlap)))\r\n        \r\n        scores_sorted = sorted(scores, key=lambda x: x[2], reverse=True)\r\n        \r\n        dataset_results = []\r\n        publication_results = []\r\n        \r\n        for key, entry_type, sc, simv, ov in scores_sorted:\r\n            if entry_type == \"dataset\" and sc >= SIMILARITY_THRESHOLD and len(dataset_results) < top_k_datasets:\r\n                dataset_results.append({\"dataset\": key, \"score\": sc, \"sim\": simv, \"overlap\": ov})\r\n            elif entry_type == \"publication\" and sc >= SIMILARITY_THRESHOLD and len(publication_results) < top_k_publications:\r\n                pub_idx = int(key.replace(\"pub_\", \"\"))\r\n                if pub_idx < len(self.publications):\r\n                    pub = self.publications[pub_idx]\r\n                    publication_results.append({\r\n                        \"title\": pub.get(\"Title\", \"\"),\r\n                        \"link\": pub.get(\"Link\", \"\"),\r\n                        \"score\": sc,\r\n                        \"sim\": simv,\r\n                        \"overlap\": ov\r\n                    })\r\n\r\n        return dataset_results, publication_results\r\n\r\n\r\n# ---------- Dataset context builder ----------\r\ndef build_context_for_dataset(ds: str, meta: Dict[str, Any], max_files: int = 4,\r\n                              analysis_rows: int = ANALYSIS_ROWS, preview_rows: int = PREVIEW_ROWS) -> Tuple[str, List[str], Dict[str, Any]]:\r\n    urls = []\r\n    blocks = []\r\n    structured = {\"dataset\": ds, \"columns\": {}}\r\n\r\n    desc = meta.get(\"description\", \"\")\r\n    structured[\"description\"] = desc\r\n    if desc:\r\n        urls.extend(extract_urls(str(desc)))\r\n        blocks.append(f\"Description: {desc}\")\r\n\r\n    files = meta.get(\"files\", {}) if isinstance(meta, dict) else {}\r\n    file_count = 0\r\n    for name, entry in list(files.items())[:max_files]:\r\n        path = Path(entry if isinstance(entry, str) else (entry[0] if entry else \"\"))\r\n        if not path.exists():\r\n            blocks.append(f\"FILE: {name} - (missing)\")\r\n            file_count += 1\r\n            continue\r\n        \r\n        df_analysis = read_table_for_analysis(path, nrows=analysis_rows)\r\n        df_preview = safe_read_table(path, nrows=preview_rows)\r\n        if df_preview is not None:\r\n            blocks.append(f\"FILE PREVIEW: {path.name}\\n{df_preview.head(preview_rows).to_string(index=False)}\")\r\n        else:\r\n            blocks.append(f\"FILE: {path.name} (preview unreadable)\")\r\n\r\n        if df_analysis is None or df_analysis.empty:\r\n            try:\r\n                txt = path.read_text(encoding=\"utf-8\", errors=\"ignore\")[:2000]\r\n                urls.extend(extract_urls(txt))\r\n            except Exception:\r\n                pass\r\n            file_count += 1\r\n            continue\r\n\r\n        col_summ = analyze_dataframe(df_analysis, top_n=6)\r\n        structured[\"columns\"].update(col_summ)\r\n        for col, info in col_summ.items():\r\n            if info.get(\"count\", 0) == 0:\r\n                continue\r\n            if info.get(\"numeric\"):\r\n                top_vals = info.get(\"top\", [])\r\n                top_str = \", \".join(f\"{tv['value']} ({tv['pct']}%)\" for tv in top_vals[:4]) if top_vals else \"\"\r\n                blocks.append(f\"Column: {col} [numeric] count={info.get('count')} mean={info.get('mean'):.3f} median={info.get('median'):.3f} std={info.get('std'):.3f} top={top_str}\")\r\n                for tv in top_vals:\r\n                    urls.extend(extract_urls(str(tv.get(\"value\"))))\r\n            else:\r\n                top_vals = info.get(\"top\", [])\r\n                top_str = \", \".join(f\"'{tv['value']}' ({tv['pct']}%)\" for tv in top_vals[:6]) if top_vals else \"\"\r\n                blocks.append(f\"Column: {col} [categorical] count={info.get('count')} unique={info.get('unique')} top={top_str}\")\r\n                for tv in top_vals:\r\n                    urls.extend(extract_urls(str(tv.get(\"value\"))))\r\n        file_count += 1\r\n\r\n    context_text = \"\\n\".join(blocks)\r\n    deduped_urls = []\r\n    for u in urls:\r\n        if u not in deduped_urls:\r\n            deduped_urls.append(u)\r\n    return context_text, deduped_urls, structured\r\n\r\n\r\n# ---------- LongCat API Integration ----------\r\ndef call_longcat(prompt: str) -> Tuple[bool, Any]:\r\n    \"\"\"\r\n    Connects to LongCat AI via the OpenAI SDK compatibility layer.\r\n    \"\"\"\r\n    # 1. Get API Key\r\n    api_key = st.secrets.get(\"LONGCAT_API_KEY\", DEFAULT_LONGCAT_KEY)\r\n    \r\n    if not api_key:\r\n        return False, \"LongCat API key missing. Please check secrets or configuration.\"\r\n\r\n    # 2. Initialize Client\r\n    try:\r\n        client = OpenAI(\r\n            api_key=api_key,\r\n            base_url=LONGCAT_BASE_URL\r\n        )\r\n    except Exception as e:\r\n        return False, f\"Failed to initialize OpenAI/LongCat Client: {e}\"\r\n\r\n    # 3. Call API\r\n    attempt = 0\r\n    while attempt < MAX_API_ATTEMPTS:\r\n        attempt += 1\r\n        try:\r\n            logger.info(\"LongCat call attempt %d\", attempt)\r\n            \r\n            response = client.chat.completions.create(\r\n                model=LONGCAT_MODEL,\r\n                messages=[\r\n                    {\"role\": \"system\", \"content\": \"You are a helpful NASA Space Biology expert assistant.\"},\r\n                    {\"role\": \"user\", \"content\": prompt}\r\n                ],\r\n                max_tokens=1024,\r\n                temperature=0.7\r\n            )\r\n            return True, response\r\n        except Exception as e:\r\n            logger.warning(\"LongCat attempt %d failed: %s\", attempt, e)\r\n            if attempt >= MAX_API_ATTEMPTS:\r\n                return False, f\"LongCat failed after {attempt} attempts: {e}\"\r\n            time.sleep(1.5 * attempt)\r\n            \r\n    return False, \"API attempts exhausted\"\r\n\r\n\r\ndef parse_longcat_response(resp: Any) -> Tuple[str, List[str], Dict[str, Any]]:\r\n    \"\"\"\r\n    Parses the OpenAI-style response object from LongCat.\r\n    \"\"\"\r\n    try:\r\n        # Extract content from the ChatCompletion object\r\n        if hasattr(resp, 'choices') and len(resp.choices) > 0:\r\n            content = resp.choices[0].message.content\r\n        else:\r\n            # Fallback for weird responses or errors passed as strings\r\n            content = str(resp)\r\n\r\n        if not content:\r\n            content = \"No content returned from API.\"\r\n\r\n        # Extract URLs from the answer text\r\n        urls = extract_urls(content)\r\n        \r\n        # Meta info\r\n        meta = {\r\n            \"model\": getattr(resp, 'model', 'unknown'),\r\n            \"usage\": dict(resp.usage) if hasattr(resp, 'usage') else {}\r\n        }\r\n        \r\n        return content, list(dict.fromkeys(urls)), meta\r\n\r\n    except Exception as e:\r\n        logger.exception(\"Parse error\")\r\n        return str(resp), extract_urls(str(resp)), {\"parse_error\": str(e)}\r\n\r\n\r\ndef clean_answer_text(text: str) -> str:\r\n    if not text:\r\n        return \"\"\r\n    t = re.sub(r\"[\\x00-\\x1f]\", \" \", text)\r\n    t = re.sub(r\"As an AI language model[^\\n.]*[.\\n]?\", \"\", t, flags=re.I)\r\n    t = re.sub(r\"\\s{2,}\", \" \", t)\r\n    return t.strip()\r\n\r\n\r\n# ---------- Fallback summarizer ----------\r\ndef local_fallback_summary(question: str, context_text: str, candidate_urls: List[str], publications: List[Dict[str, Any]]) -> str:\r\n    lines = [l.strip() for l in (context_text or \"\").splitlines() if l.strip()]\r\n    important = [l for l in lines if any(k in l.lower() for k in (\"flight\", \"space\", \"microgravity\", \"orbit\", \"condition\", \"control\", \"assay\", \"organism\", \"transcript\", \"methyl\", \"gene\", \"expression\", \"rna\", \"dna\"))]\r\n    if not important:\r\n        important = lines[:6]\r\n    summary = f\"Local fallback summary for: {question}\\n\"\r\n    if important:\r\n        summary += \"\\n\".join(f\"- {l}\" for l in important[:8])\r\n    else:\r\n        summary += \"No local data available.\"\r\n    \r\n    if publications:\r\n        summary += \"\\n\\nRelated Research Publications:\\n\"\r\n        for pub in publications[:5]:\r\n            summary += f\"- {pub.get('title', '')}\\n  {pub.get('link', '')}\\n\"\r\n    \r\n    if candidate_urls:\r\n        summary += \"\\n\\nLocal dataset URLs:\\n\" + \"\\n\".join(candidate_urls[:6])\r\n    \r\n    summary += \"\\n\\nSuggested NASA data sources to search:\\n\"\r\n    nasa_keywords = [\"NASA space biology\", \"microgravity research\", \"spaceflight transcriptomics\", \"NASA gene expression data\"]\r\n    for kw in nasa_keywords:\r\n        if any(word in question.lower() for word in kw.split()):\r\n            summary += f\"- Search Google for: '{kw} dataset NASA Open Science'\\n\"\r\n    \r\n    return summary\r\n\r\n\r\n# ---------- High-level ask flow ----------\r\nclass EngineRunner:\r\n    def __init__(self, data_index: Dict[str, Any], publications: List[Dict[str, str]]):\r\n        self.data_index = data_index or {}\r\n        self.publications = publications or []\r\n        self.search_index = SearchIndex(self.data_index, self.publications)\r\n\r\n    def rebuild_search_index(self):\r\n        self.search_index.rebuild(self.data_index, self.publications)\r\n\r\n    def ask(self, question: str, prefer_dataset_urls_first: bool = True) -> Dict[str, Any]:\r\n        # find relevant datasets\r\n        dataset_candidates, publication_candidates = self.search_index.query(\r\n            question, \r\n            top_k_datasets=TOP_K_DATASETS, \r\n            top_k_publications=TOP_K_PUBLICATIONS\r\n        )\r\n        \r\n        used_datasets = dataset_candidates\r\n        used_publications = publication_candidates\r\n        \r\n        contexts = []\r\n        dataset_urls = []\r\n        structured = {}\r\n        \r\n        # Build context for datasets\r\n        for c in dataset_candidates:\r\n            ds = c[\"dataset\"]\r\n            meta = self.data_index.get(ds, {}) if isinstance(self.data_index, dict) else {}\r\n            ctx, urls, struct = build_context_for_dataset(ds, meta)\r\n            contexts.append(f\"--- Dataset: {ds} (score={c['score']:.3f}) ---\\n{ctx}\")\r\n            dataset_urls += urls\r\n            structured[ds] = struct\r\n\r\n        # Add publications to context\r\n        if publication_candidates:\r\n            contexts.append(\"--- Related Research Publications ---\")\r\n            for pub in publication_candidates:\r\n                contexts.append(f\"Publication: {pub['title']}\\nLink: {pub['link']}\\nRelevance Score: {pub['score']:.3f}\")\r\n\r\n        combined_context = \"\\n\\n\".join(contexts).strip()\r\n        dataset_urls = list(dict.fromkeys(dataset_urls))\r\n\r\n        use_web_search = not bool(combined_context)\r\n\r\n        # Prompt for LongCat\r\n        prompt = (\r\n            \"SYSTEM: You are a NASA Space Biology expert assistant. \"\r\n            \"Follow these guidelines precisely:\\n\\n\"\r\n            \"1. FIRST analyze the provided DATA CONTEXT including datasets and research publications\\n\"\r\n            \"2. If DATA CONTEXT contains relevant information, use it to answer the question\\n\"\r\n            \"3. If DATA CONTEXT is insufficient, suggest targeted searches for:\\n\"\r\n            \"   - NASA Open Science Data Repository (data.nasa.gov)\\n\"\r\n            \"   - NASA GeneLab (genelab.nasa.gov)\\n\" \r\n            \"   - PubMed Central\\n\\n\"\r\n            \"4. Answer structure:\\n\"\r\n            \"   - Start with direct answer (2-3 sentences max)\\n\"\r\n            \"   - Mention which datasets and publications were used (if any)\\n\"\r\n            \"   - List 3-5 most relevant URLs with brief descriptions\\n\"\r\n            \"   - Include specific NASA data sources when applicable\\n\\n\"\r\n            \"5. CREDIBILITY: Prioritize .gov, .edu, NASA, and peer-reviewed sources\\n\"\r\n            \"6. FORMAT: Keep answer concise (<200 words), no JSON in visible answer\\n\\n\"\r\n            f\"DATA CONTEXT:\\n{combined_context or '[NO RELEVANT LOCAL DATASETS OR PUBLICATIONS FOUND]'}\\n\\n\"\r\n            f\"USER QUESTION:\\n{question}\\n\\n\"\r\n            \"Provide a focused answer with NASA data recommendations and relevant publications.\"\r\n        )\r\n        if use_web_search:\r\n            prompt += \"\\n\\nNote: No local data found. Please rely on your general knowledge and suggest external NASA resources.\"\r\n\r\n        ok, resp = call_longcat(prompt)\r\n        if not ok:\r\n            logger.warning(\"LongCat API failed: %s\", resp)\r\n            fallback = local_fallback_summary(question, combined_context, dataset_urls, used_publications)\r\n            return {\r\n                \"answer\": fallback,\r\n                \"recommended_urls\": dataset_urls,\r\n                \"used_datasets\": used_datasets,\r\n                \"used_publications\": used_publications,\r\n                \"meta\": {\"fallback\": True, \"error\": str(resp)},\r\n                \"structured\": structured\r\n            }\r\n\r\n        text, urls_from_resp, meta = parse_longcat_response(resp)\r\n        text = clean_answer_text(text)\r\n        \r\n        # Combine URLs\r\n        combined_urls = []\r\n        if prefer_dataset_urls_first:\r\n            for u in dataset_urls:\r\n                if u not in combined_urls:\r\n                    combined_urls.append(u)\r\n            for pub in used_publications:\r\n                if pub.get(\"link\") and pub[\"link\"] not in combined_urls:\r\n                    combined_urls.append(pub[\"link\"])\r\n            for u in urls_from_resp:\r\n                if u not in combined_urls:\r\n                    combined_urls.append(u)\r\n        else:\r\n            for u in urls_from_resp:\r\n                if u not in combined_urls:\r\n                    combined_urls.append(u)\r\n            for pub in used_publications:\r\n                if pub.get(\"link\") and pub[\"link\"] not in combined_urls:\r\n                    combined_urls.append(pub[\"link\"])\r\n            for u in dataset_urls:\r\n                if u not in combined_urls:\r\n                    combined_urls.append(u)\r\n                    \r\n        return {\r\n            \"answer\": text,\r\n            \"recommended_urls\": combined_urls,\r\n            \"used_datasets\": used_datasets,\r\n            \"used_publications\": used_publications,\r\n            \"meta\": meta,\r\n            \"structured\": structured\r\n        }\r\n\r\n\r\n# ---------- Helpers to load index and publications ----------\r\ndef load_index() -> Dict[str, Any]:\r\n    if not INDEX_PATH.exists():\r\n        logger.warning(\"index.json not found; App will start with empty index.\")\r\n        return {}\r\n    try:\r\n        with INDEX_PATH.open(\"r\", encoding=\"utf-8\") as fh:\r\n            return json.load(fh)\r\n    except Exception as e:\r\n        logger.exception(\"Failed to load index.json\")\r\n        return {}\r\n\r\n\r\ndef load_publications() -> List[Dict[str, str]]:\r\n    if not PUBLICATIONS_PATH.exists():\r\n        logger.warning(\"SB_publication_PMC.json not found; App will start without publications.\")\r\n        return []\r\n    try:\r\n        with PUBLICATIONS_PATH.open(\"r\", encoding=\"utf-8\") as fh:\r\n            return json.load(fh)\r\n    except Exception as e:\r\n        logger.exception(\"Failed to load SB_publication_PMC.json\")\r\n        return []\r\n\r\n\r\n# ---------- Streamlit App ----------\r\ndef main():\r\n    st.set_page_config(\r\n        page_title=\"Nasa Biological Space Engine — Intelligent NASA Space Biology Search\",\r\n        page_icon=\"🚀\",\r\n        layout=\"wide\",\r\n        initial_sidebar_state=\"expanded\"\r\n    )\r\n    \r\n    if 'runner' not in st.session_state:\r\n        data_index = load_index()\r\n        publications = load_publications()\r\n        st.session_state.runner = EngineRunner(data_index, publications)\r\n        st.session_state.logs = [\"App initialized. Intelligent NASA Space Biology search engine loaded.\"]\r\n        st.session_state.current_result = None\r\n        st.session_state.selected_dataset = None\r\n        \r\n    \r\n    st.title(\"Nasa Biological Space Engine \")\r\n    st.subheader(\"Intelligent NASA Space Biology Search \")\r\n    \r\n    with st.sidebar:\r\n        st.header(\"Controls & Information\")\r\n        if st.button(\"🔁 Rebuild Search Index\", use_container_width=True):\r\n            with st.spinner(\"Rebuilding NASA space biology search index...\"):\r\n                try:\r\n                    st.session_state.runner.rebuild_search_index()\r\n                    st.session_state.logs.append(\"Search index rebuilt.\")\r\n                    st.success(\"NASA space biology search index rebuilt.\")\r\n                except Exception as e:\r\n                    st.error(f\"Could not rebuild index: {e}\")\r\n                    st.session_state.logs.append(f\"Rebuild failed: {e}\")\r\n        \r\n        st.divider()\r\n        st.subheader(\"📊 Matched Datasets\")\r\n        if st.session_state.current_result:\r\n            datasets = st.session_state.current_result.get(\"used_datasets\", [])\r\n            dataset_options = [f\"{d['dataset']} (score={d['score']:.3f})\" for d in datasets]\r\n            if dataset_options:\r\n                selected = st.selectbox(\"Select dataset for preview:\", dataset_options)\r\n                if selected:\r\n                    st.session_state.selected_dataset = selected.split()[0]\r\n            else:\r\n                st.info(\"No datasets matched\")\r\n        else:\r\n            st.info(\"Run a query to see matched datasets\")\r\n        \r\n        st.divider()\r\n        st.subheader(\"📝 Logs\")\r\n        log_container = st.container(height=300)\r\n        with log_container:\r\n            for log_entry in st.session_state.logs[-10:]:\r\n                st.text(log_entry)\r\n    \r\n    col1, col2 = st.columns([3, 1])\r\n    with col1:\r\n        query = st.text_input(\r\n            \"🔍 Ask NASA Space Biology Question:\",\r\n            placeholder=\"e.g., 'What are the effects of microgravity on gene expression in mice?'\",\r\n            key=\"query_input\"\r\n        )\r\n    with col2:\r\n        st.write(\"\")\r\n        st.write(\"\")\r\n        ask_btn = st.button(\"🚀 Ask\", use_container_width=True)\r\n    \r\n    if ask_btn and query:\r\n        with st.spinner(\"🔬 Analyzing question for NASA space biology relevance...\"):\r\n            st.session_state.current_result = None\r\n            st.session_state.selected_dataset = None\r\n            st.session_state.logs.append(f\"Query: {query}\")\r\n            \r\n            with st.status(\"Searching NASA space biology datasets and publications...\", expanded=True) as status:\r\n                dataset_cand, publication_cand = st.session_state.runner.search_index.query(\r\n                    query, \r\n                    top_k_datasets=TOP_K_DATASETS, \r\n                    top_k_publications=TOP_K_PUBLICATIONS\r\n                )\r\n                st.write(\"**Datasets:**\")\r\n                for c in dataset_cand:\r\n                    st.write(f\"• {c['dataset']} (score: {c['score']:.3f})\")\r\n                st.write(\"**Publications:**\")\r\n                for c in publication_cand[:3]:\r\n                    st.write(f\"• {c['title'][:80]}... (score: {c['score']:.3f})\")\r\n                status.update(label=\"Found relevant datasets and publications\", state=\"complete\")\r\n            \r\n            with st.status(\"Consulting LongCat AI and analyzing data...\", expanded=True) as status:\r\n                result = st.session_state.runner.ask(query, prefer_dataset_urls_first=True)\r\n                st.session_state.current_result = result\r\n                st.session_state.logs.append(\"NASA space biology analysis complete.\")\r\n                status.update(label=\"Analysis complete\", state=\"complete\")\r\n    \r\n    if st.session_state.current_result:\r\n        result = st.session_state.current_result\r\n        tab1, tab2, tab3, tab4, tab5 = st.tabs([\"💬 Answer\", \"🔗 Recommended URLs\", \"📚 Research Publications\", \"📊 Dataset Details\", \"📋 Raw Metadata\"])\r\n        \r\n        with tab1:\r\n            st.subheader(\"Answer\")\r\n            st.write(result.get(\"answer\", \"\"))\r\n            if st.button(\"📋 Copy Answer\", key=\"copy_answer\"):\r\n                st.code(result.get(\"answer\", \"\"), language=None)\r\n                st.success(\"Answer copied to clipboard!\")\r\n        \r\n        with tab2:\r\n            st.subheader(\"Recommended URLs\")\r\n            urls = result.get(\"recommended_urls\", [])\r\n            if urls:\r\n                for i, url in enumerate(urls):\r\n                    col1, col2 = st.columns([4, 1])\r\n                    with col1:\r\n                        st.write(f\"{i+1}. {url}\")\r\n                    with col2:\r\n                        if st.button(\"🌐 Open\", key=f\"open_{i}\"):\r\n                            webbrowser.open_new_tab(url)\r\n            else:\r\n                st.info(\"No URLs recommended\")\r\n        \r\n        with tab3:\r\n            st.subheader(\"Research Publications\")\r\n            publications = result.get(\"used_publications\", [])\r\n            if publications:\r\n                for i, pub in enumerate(publications):\r\n                    with st.expander(f\"📄 {pub.get('title', 'Unknown Title')}\", expanded=i < 2):\r\n                        st.write(f\"**Title:** {pub.get('title', 'N/A')}\")\r\n                        st.write(f\"**Link:** {pub.get('link', 'N/A')}\")\r\n                        st.write(f\"**Relevance Score:** {pub.get('score', 0):.3f}\")\r\n                        col1, col2 = st.columns([1, 1])\r\n                        with col1:\r\n                            if st.button(\"🌐 Open Publication\", key=f\"open_pub_{i}\"):\r\n                                webbrowser.open_new_tab(pub.get('link', ''))\r\n                        with col2:\r\n                            if st.button(\"📋 Copy Link\", key=f\"copy_pub_{i}\"):\r\n                                st.code(pub.get('link', ''), language=None)\r\n                                st.success(\"Link copied to clipboard!\")\r\n            else:\r\n                st.info(\"No relevant publications found\")\r\n        \r\n        with tab4:\r\n            st.subheader(\"Dataset Details\")\r\n            if st.session_state.selected_dataset:\r\n                ds = st.session_state.selected_dataset\r\n                meta = st.session_state.runner.data_index.get(ds, {})\r\n                with st.expander(f\"📁 Dataset: {ds}\", expanded=True):\r\n                    ctx, urls, struct = build_context_for_dataset(ds, meta)\r\n                    st.subheader(\"Context Summary\")\r\n                    st.text_area(\"Context\", ctx, height=200, key=f\"ctx_{ds}\")\r\n                    if urls:\r\n                        st.subheader(\"Dataset URLs\")\r\n                        for url in urls:\r\n                            st.write(f\"• {url}\")\r\n                    st.subheader(\"Column Summaries\")\r\n                    st.json(struct)\r\n            else:\r\n                st.info(\"Select a dataset from the sidebar to view details\")\r\n        \r\n        with tab5:\r\n            st.subheader(\"Raw Metadata\")\r\n            out_meta = {\r\n                \"meta\": result.get(\"meta\", {}),\r\n                \"used_datasets\": result.get(\"used_datasets\", []),\r\n                \"used_publications\": result.get(\"used_publications\", []),\r\n                \"structured\": result.get(\"structured\", {})\r\n            }\r\n            st.json(out_meta)\r\n            json_str = json.dumps(out_meta, indent=2, ensure_ascii=False)\r\n            b64 = base64.b64encode(json_str.encode()).decode()\r\n            href = f'<a href=\"data:file/json;base64,{b64}\" download=\"nasa_space_biology_result.json\">💾 Download Result as JSON</a>'\r\n            st.markdown(href, unsafe_allow_html=True)\r\n    \r\n    elif st.session_state.selected_dataset and not st.session_state.current_result:\r\n        ds = st.session_state.selected_dataset\r\n        meta = st.session_state.runner.data_index.get(ds, {})\r\n        st.subheader(f\"📁 Dataset Preview: {ds}\")\r\n        ctx, urls, struct = build_context_for_dataset(ds, meta)\r\n        with st.expander(\"Context Summary\", expanded=True):\r\n            st.text_area(\"Context\", ctx, height=200)\r\n        if urls:\r\n            with st.expander(\"Dataset URLs\", expanded=True):\r\n                for url in urls:\r\n                    st.write(f\"• {url}\")\r\n        with st.expander(\"Column Summaries\", expanded=True):\r\n            st.json(struct)\r\n\r\n\r\nif __name__ == \"__main__\":\r\n\r\n    main()\r\n"}}}}