1 line
104 KiB
JSON
1 line
104 KiB
JSON
{"repo": "anikethh/iris-interactive-research-ideation-system", "files": {"app.py": {"d1ed922c5a2342dab2316dded836671f6aae6a64": {"t": 1785674057.2792509, "n": 101867, "h": "bd42644bd8d6aa48", "c": "from flask import Flask, jsonify, request, render_template\nfrom flask_socketio import SocketIO, emit\nimport os\nimport random\nimport math # Add math module for UCT calculations\nfrom src.mcts.node import MCTSState, MCTSNode\nfrom src.mcts.tree import MCTS\nfrom pathlib import Path\nfrom werkzeug.utils import secure_filename\nimport uuid\nfrom src.agents.structured_review import StructuredReviewAgent\nfrom src.agents.ideation import IdeationAgent\nfrom src.agents.review import ReviewAgent\nimport json\nimport re\nimport yaml\nimport traceback\nimport sys\nimport logging\nimport click\n\nlogger = logging.getLogger(__name__)\n\n# Remove the old external path and use local scholarqa package\nsys.path.insert(0, str(Path(__file__).parent / \"src\" / \"retrieval_api\"))\nfrom scholarqa import ScholarQA\nfrom scholarqa.rag.retrieval import PaperFinder, PaperFinderWithReranker\nfrom scholarqa.rag.retriever_base import FullTextRetriever\nfrom scholarqa.rag.reranker.modal_engine import ModalReranker\nfrom scholarqa.rag.reranker.modal_engine import HuggingFaceReranker\nimport pymupdf # PyMuPDF for PDF parsing\n# Import the key manager\n# from src.utils.key_manager import encrypt_api_key, decrypt_api_key, get_client_encryption_script\n\napp = Flask(__name__)\napp.config[\"UPLOAD_FOLDER\"] = \"uploads\"\napp.config[\"MAX_CONTENT_LENGTH\"] = 16 * 1024 * 1024 # 16MB max file size\napp.config['SECRET_KEY'] = os.environ.get('FLASK_SECRET_KEY', 'secret!')\nsocketio = SocketIO(app)\n\n# Create uploads directory if it doesn't exist\nos.makedirs(app.config[\"UPLOAD_FOLDER\"], exist_ok=True)\n\n# Create a secure key directory for storing encrypted keys if not exists\nSECURE_KEYS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'secure_keys')\nos.makedirs(SECURE_KEYS_DIR, exist_ok=True)\n\n# Initialize knowledge storage\nknowledge_chunks = []\n\n# Initialize empty storage\nchat_messages = []\nmain_idea = \"Generating Research Idea...\"\nretrieval_results = {}\n\ndef allowed_file(filename):\n \"\"\"Check if the file extension is allowed\"\"\"\n # Get file extension (last part after the dot)\n extension = filename.rsplit(\".\", 1)[1].lower() if \".\" in filename else \"\"\n return extension in {\n \"txt\",\n \"pdf\",\n \"doc\",\n \"docx\",\n }\n\n# Function to safely load API keys from environment or config file\ndef get_api_key(key_name, config_dict):\n \"\"\"Get API key from environment variable with preference to environment variable\"\"\"\n env_var_name = f\"{key_name.upper()}_API_KEY\"\n env_key = os.environ.get(env_var_name)\n \n if env_key:\n return env_key\n \n # Never return sensitive keys from config in production\n if os.environ.get('FLASK_ENV') == 'production':\n return None\n \n # Only fall back to config file in development\n if \"keys\" in config_dict and key_name in config_dict[\"keys\"]:\n return config_dict[\"keys\"][key_name]\n \n return None\n\ndef extract_abstract(pdf_text):\n full_text = \"\\n\".join(pdf_text)\n match = re.search(r\"(Abstract.*?)(\\n(?:Introduction|Keywords|1\\.)|\\n{2,})\", full_text, re.DOTALL | re.IGNORECASE)\n if match:\n return match.group(1).strip()\n return \"No abstract found.\"\n\n# Initialize MCTS\nmcts = MCTS(\"config/config.yaml\")\ncurrent_root = None\ncurrent_node = None\ncurrent_state = None\nexploration_in_progress = False\n\n# Initialize agents\nstructured_review_agent = StructuredReviewAgent(\"config/config.yaml\")\nideation_agent = IdeationAgent(\"config/config.yaml\")\nreview_agent = ReviewAgent(\"config/config.yaml\") # Initialize review agent directly\n\n# Get the configuration from config.yaml\nimport yaml\nwith open(\"config/config.yaml\", \"r\") as f:\n config = yaml.safe_load(f)\n\n# Set Semantic Scholar API key\ns2_api_key = os.environ.get(\"SEMANTIC_SCHOLAR_API_KEY\")\nif not s2_api_key:\n logger.warning(\"SEMANTIC_SCHOLAR_API_KEY environment variable is not set. Semantic Scholar API features will be disabled.\")\n os.environ['SEMANTIC_SCHOLAR_API_KEY'] = ''\nelse:\n os.environ['SEMANTIC_SCHOLAR_API_KEY'] = s2_api_key\n\n# Set Google API key securely only if not in Azure OpenAI deployment mode\ndeploy_mode = os.environ.get('DEPLOY', 'false').lower() == 'true'\nif not deploy_mode:\n gemini_key = get_api_key(\"googleaistudio\", config)\n if gemini_key:\n os.environ['GEMINI_API_KEY'] = gemini_key\nelse:\n # In Azure OpenAI mode, ensure Azure OpenAI keys are available\n azure_key = os.environ.get(\"AZURE_OPENAI_API_KEY\")\n azure_endpoint = os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n if not azure_key:\n logger.warning(\"AZURE_OPENAI_API_KEY not set while in deployment mode\")\n if not azure_endpoint:\n logger.warning(\"AZURE_OPENAI_ENDPOINT not set while in deployment mode\")\n\nretriever = FullTextRetriever(n_retrieval=10, n_keyword_srch=10)\n# paper_finder = PaperFinder(retriever, context_threshold=0.1)\nreranker = HuggingFaceReranker(model_name=\"cross-encoder/ms-marco-MiniLM-L-6-v2\", batch_size=256)\npaper_finder = PaperFinderWithReranker(retriever, reranker=reranker, n_rerank=5, context_threshold=0.1)\nscholar_qa = ScholarQA(paper_finder=paper_finder, llm_model=\"gemini/gemini-2.0-flash-lite\")\n\n# API Key management endpoints with improved security\n@app.route(\"/api/set_api_key\", methods=[\"POST\"])\ndef set_api_key():\n \"\"\"Save an API key securely\"\"\"\n try:\n # Verify request contains required data\n data = request.get_json()\n if not data or \"provider\" not in data or \"key\" not in data:\n return jsonify({\"error\": \"Missing required fields\"}), 400\n \n provider = data[\"provider\"]\n api_key = data[\"key\"]\n \n # Validate API key format based on provider\n key_formats = {\n \"openai\": r\"^sk-[A-Za-z0-9]{48}$\",\n \"claude\": r\"^sk-ant-[A-Za-z0-9]{32,}$\",\n \"deepseek\": r\"^[A-Za-z0-9]{32,}$\",\n \"gemini\": r\"^AIza[A-Za-z0-9_-]{35}$\",\n \"semantic_scholar\": r\"^[A-Za-z0-9]{40}$\"\n }\n \n if provider in key_formats:\n if not re.match(key_formats[provider], api_key):\n return jsonify({\"error\": f\"Invalid {provider} API key format\"}), 400\n \n # Encrypt the API key if not in local development mode\n is_production = os.environ.get('FLASK_ENV') == 'production'\n \n # Read the current config for local development\n with open(\"config/config.yaml\", \"r\") as f:\n config = yaml.safe_load(f)\n \n # Ensure the keys section exists\n if \"keys\" not in config:\n config[\"keys\"] = {}\n \n # Set environment variable immediately for current session\n provider_key_mapping = {\n \"openai\": \"OPENAI_API_KEY\",\n \"claude\": \"ANTHROPIC_API_KEY\",\n \"deepseek\": \"DEEPSEEK_API_KEY\",\n \"gemini\": \"GEMINI_API_KEY\",\n \"semantic_scholar\": \"SEMANTIC_SCHOLAR_API_KEY\"\n }\n \n # Save the key to environment variable for immediate use\n if provider in provider_key_mapping:\n os.environ[provider_key_mapping[provider]] = api_key\n \n # In production, encrypt keys for storage\n if is_production:\n # Use app secret key as encryption password\n encrypted_data = encrypt_api_key(api_key, app.config['SECRET_KEY'])\n \n # Store encrypted key by provider\n encrypted_key_path = os.path.join(SECURE_KEYS_DIR, f\"{provider}.json\")\n with open(encrypted_key_path, 'w') as f:\n json.dump(encrypted_data, f)\n \n return jsonify({\n \"status\": \"success\", \n \"message\": f\"API key for {provider} saved securely\"\n })\n else:\n # For local development, save in config file\n if provider == \"openai\":\n config[\"keys\"][\"openai_key\"] = api_key\n elif provider == \"claude\":\n config[\"keys\"][\"anthropic_key\"] = api_key\n elif provider == \"deepseek\":\n config[\"keys\"][\"deepseek_key\"] = api_key\n elif provider == \"gemini\":\n config[\"keys\"][\"googleaistudio_key\"] = api_key\n \n # Save the updated config\n with open(\"config/config.yaml\", \"w\") as f:\n yaml.dump(config, f, default_flow_style=False)\n \n return jsonify({\n \"status\": \"success\", \n \"message\": f\"API key for {provider} saved successfully\"\n })\n \n except Exception as e:\n error_message = f\"Error saving API key: {str(e)}\"\n print(error_message)\n return jsonify({\"error\": error_message}), 500\n\n@app.route(\"/api/delete_api_key\", methods=[\"POST\"])\ndef delete_api_key():\n \"\"\"Delete an API key\"\"\"\n try:\n data = request.get_json()\n if not data or \"provider\" not in data:\n return jsonify({\"error\": \"Missing provider field\"}), 400\n \n provider = data[\"provider\"]\n is_production = os.environ.get('FLASK_ENV') == 'production'\n \n if is_production:\n # For production, remove the encrypted key file\n encrypted_key_path = os.path.join(SECURE_KEYS_DIR, f\"{provider}.json\")\n if os.path.exists(encrypted_key_path):\n os.remove(encrypted_key_path)\n return jsonify({\n \"status\": \"success\", \n \"message\": f\"API key for {provider} deleted successfully\"\n })\n else:\n return jsonify({\n \"status\": \"error\", \n \"message\": f\"No API key found for {provider}\"\n }), 404\n else:\n # For local development, remove from config file\n with open(\"config/config.yaml\", \"r\") as f:\n config = yaml.safe_load(f)\n \n # Check if keys section exists\n if \"keys\" not in config:\n return jsonify({\"status\": \"error\", \"message\": \"No API keys found\"}), 404\n \n # Remove appropriate key based on provider\n key_removed = False\n if provider == \"openai\" and \"openai_key\" in config[\"keys\"]:\n del config[\"keys\"][\"openai_key\"]\n key_removed = True\n elif provider == \"claude\" and \"anthropic_key\" in config[\"keys\"]:\n del config[\"keys\"][\"anthropic_key\"]\n key_removed = True\n elif provider == \"deepseek\" and \"deepseek_key\" in config[\"keys\"]:\n del config[\"keys\"][\"deepseek_key\"]\n key_removed = True\n elif provider == \"gemini\" and \"googleaistudio_key\" in config[\"keys\"]:\n del config[\"keys\"][\"googleaistudio_key\"]\n key_removed = True\n \n if key_removed:\n # Save the updated config\n with open(\"config/config.yaml\", \"w\") as f:\n yaml.dump(config, f, default_flow_style=False)\n return jsonify({\n \"status\": \"success\", \n \"message\": f\"API key for {provider} deleted successfully\"\n })\n else:\n return jsonify({\n \"status\": \"error\", \n \"message\": f\"No API key found for {provider}\"\n }), 404\n \n except Exception as e:\n error_message = f\"Error deleting API key: {str(e)}\"\n print(error_message)\n return jsonify({\"error\": error_message}), 500\n\n@app.route(\"/api/get_api_keys\", methods=[\"GET\"])\ndef get_api_keys():\n \"\"\"Get a list of configured API keys (without exposing the actual key values)\"\"\"\n try:\n is_production = os.environ.get('FLASK_ENV') == 'production'\n configured_keys = []\n \n if is_production:\n # In production, check for encrypted key files\n for provider in [\"openai\", \"claude\", \"deepseek\", \"gemini\"]:\n encrypted_key_path = os.path.join(SECURE_KEYS_DIR, f\"{provider}.json\")\n if os.path.exists(encrypted_key_path):\n configured_keys.append({\n \"provider\": provider,\n \"configured\": True\n })\n else:\n # In local development, check the config file\n with open(\"config/config.yaml\", \"r\") as f:\n config = yaml.safe_load(f)\n \n # Check if keys section exists\n if \"keys\" in config:\n # Create a list of configured providers (without exposing the actual keys)\n key_mapping = {\n \"openai_key\": \"openai\",\n \"anthropic_key\": \"claude\", \n \"deepseek_key\": \"deepseek\",\n \"googleaistudio_key\": \"gemini\"\n }\n \n for key_name, provider_name in key_mapping.items():\n if key_name in config[\"keys\"] and config[\"keys\"][key_name]:\n configured_keys.append({\n \"provider\": provider_name,\n \"configured\": True\n })\n \n return jsonify({\"keys\": configured_keys})\n \n except Exception as e:\n error_message = f\"Error retrieving API keys: {str(e)}\"\n print(error_message)\n return jsonify({\"error\": error_message}), 500\n\n# Endpoint to provide client-side encryption code\n@app.route(\"/api/encryption-script\")\ndef get_encryption_script():\n \"\"\"Return JavaScript code for client-side encryption\"\"\"\n return get_client_encryption_script(), 200, {'Content-Type': 'application/javascript'}\n\n@app.route(\"/api/llm_client_info\")\ndef get_llm_client_info():\n \"\"\"Get information about which LLM client is being used\"\"\"\n try:\n from src.agents.llm_utils import get_client_info\n client_info = get_client_info()\n \n # Add environment variable info\n env_info = {\n \"DEPLOY\": os.environ.get('DEPLOY', 'false'),\n \"AZURE_OPENAI_API_KEY_SET\": bool(os.environ.get('AZURE_OPENAI_API_KEY')),\n \"AZURE_OPENAI_ENDPOINT_SET\": bool(os.environ.get('AZURE_OPENAI_ENDPOINT')),\n \"AZURE_OPENAI_API_VERSION\": os.environ.get('AZURE_OPENAI_API_VERSION', '2024-06-01')\n }\n \n return jsonify({\n \"client_info\": client_info,\n \"environment\": env_info\n })\n \n except Exception as e:\n return jsonify({\"error\": f\"Failed to get client info: {str(e)}\"}), 500\n\n@app.route(\"/\")\ndef index():\n return render_template(\"index.html\")\n\n\n@app.route(\"/api/knowledge\", methods=[\"GET\"])\ndef get_knowledge():\n return jsonify(knowledge_chunks)\n\n\n@app.route(\"/api/add_knowledge\", methods=[\"POST\"])\ndef add_knowledge():\n data = request.get_json()\n if not data or \"text\" not in data or \"source\" not in data:\n return jsonify({\"error\": \"Invalid payload\"}), 400\n new_id = len(knowledge_chunks) + 1\n chunk = {\n \"id\": new_id,\n \"text\": data[\"text\"],\n \"full_text\": data[\"text\"],\n \"source\": data[\"source\"],\n }\n knowledge_chunks.append(chunk)\n return jsonify(chunk), 201\n\n\n@app.route(\"/api/chat\", methods=[\"GET\", \"POST\"])\ndef chat():\n global main_idea, current_root, current_node, current_state, exploration_in_progress\n if request.method == \"GET\":\n return jsonify(chat_messages)\n else:\n data = request.get_json()\n if not data or \"content\" not in data:\n return jsonify({\"error\": \"Invalid payload\"}), 400\n \n user_message = data[\"content\"]\n chat_messages.append({\"role\": \"user\", \"content\": user_message})\n \n try:\n # First message: Initialize MCTS with research goal\n if current_root is None:\n # Store initial research goal in state\n chat_messages.append(\n {\"role\": \"system\", \"content\": \"Generating initial idea...\"}\n )\n\n # Extract the abstract from the knowledge chunks (if available)\n abstract_text = \"\"\n for chunk in knowledge_chunks:\n if \"abstract\" in chunk:\n abstract_text = chunk[\"abstract\"]\n break \n \n # Create a root state that represents just the research goal\n root_state = MCTSState(\n research_goal=user_message,\n current_idea=user_message, # Root node \"idea\" is the research goal itself\n retrieved_knowledge=[abstract_text], # Pass abstract as retrieved knowledge\n feedback={},\n reward=0.0,\n depth=0,\n )\n \n # Create root node with the research goal\n current_root = MCTSNode(state=root_state)\n \n # Use the ideation agent to generate the idea\n response = mcts.ideation_agent.execute_action(\n \"generate\", \n {\n \"research_goal\": user_message,\n \"current_idea\": None,\n \"abstract\": abstract_text,\n \"action_type\": \"execute\"\n }\n )\n llm_response = response[\"content\"]\n chat_messages.append({\"role\": \"system\", \"content\": \"Initial idea generated by AI.\"})\n \n main_idea = llm_response\n \n # Create a state for the first generated idea\n first_idea_state = MCTSState(\n research_goal=user_message,\n current_idea=llm_response,\n retrieved_knowledge=[],\n feedback={},\n reward=0.0,\n depth=1, # Depth 1 since it's a child of the root\n )\n \n # Get review using the unified review method\n review_data = review_agent.unified_review(llm_response)\n print(f\"Review score: {review_data['average_score']}\")\n if review_data:\n if \"scores\" in review_data:\n first_idea_state.review_scores = review_data[\"scores\"]\n if \"reviews\" in review_data:\n first_idea_state.review_feedback = review_data[\"reviews\"]\n if \"average_score\" in review_data:\n first_idea_state.average_score = review_data[\"average_score\"]\n first_idea_state.reward = review_data[\"average_score\"]\n \n # Add the first generated idea as a child of the root node\n first_idea_node = current_root.add_child(first_idea_state, \"generate\")\n \n # Set current node to the first idea node\n current_node = first_idea_node\n \n chat_messages.append({\"role\": \"assistant\", \"content\": llm_response})\n \n # Subsequent messages: Treat as direct feedback to improve the current idea\n else:\n # Add system message indicating feedback processing\n chat_messages.append(\n {\"role\": \"system\", \"content\": \"Processing your feedback to improve the research idea...\"}\n )\n \n # Process user feedback using the ideation agent\n improved_content, raw_output = ideation_agent.process_feedback(\n idea=main_idea,\n user_feedback=user_message,\n original_raw_output=getattr(current_node.state, \"raw_llm_output\", None)\n )\n \n # Get the current feedback dictionary and add the new message\n current_feedback = current_node.state.feedback.copy() if hasattr(current_node.state, \"feedback\") else {}\n # Add the new feedback with a timestamp as key\n from datetime import datetime\n feedback_key = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n current_feedback[feedback_key] = user_message\n \n # Create a new state with updated idea based on feedback\n new_state = MCTSState(\n research_goal=current_node.state.research_goal,\n current_idea=improved_content,\n retrieved_knowledge=current_node.state.retrieved_knowledge.copy(),\n feedback=current_feedback, # Pass feedback as dictionary\n depth=current_node.state.depth + 1,\n reward=0.0 # Initial reward will be updated with review score\n )\n \n # Get review using the unified review method\n review_data = review_agent.unified_review(improved_content)\n print(f\"Review score: {review_data['average_score']}\")\n if review_data:\n if \"scores\" in review_data:\n new_state.review_scores = review_data[\"scores\"]\n if \"reviews\" in review_data:\n new_state.review_feedback = review_data[\"reviews\"]\n if \"average_score\" in review_data:\n new_state.average_score = review_data[\"average_score\"]\n new_state.reward = review_data[\"average_score\"]\n \n # Create new node and add as child of current node\n new_node = current_node.add_child(new_state, \"direct_feedback\")\n current_node = new_node\n \n # Update main idea\n main_idea = improved_content\n \n # Add system message acknowledging feedback incorporation\n chat_messages.append(\n {\"role\": \"system\", \"content\": \"Research idea updated based on your feedback.\"}\n )\n \n # Add the improved idea as an assistant message\n chat_messages.append({\"role\": \"assistant\", \"content\": improved_content})\n \n # Return the updated state\n return jsonify(\n {\n \"messages\": chat_messages,\n \"idea\": main_idea,\n \"initial_proposal\": current_root.state.research_goal if current_root else user_message,\n \"review_scores\": getattr(current_node.state, \"review_scores\", {}),\n \"average_score\": getattr(current_node.state, \"average_score\", 0.0),\n }\n )\n \n except Exception as e:\n error_message = f\"Error processing chat: {str(e)}\"\n traceback.print_exc() # Print the stack trace for debugging\n chat_messages.append({\"role\": \"system\", \"content\": error_message})\n return jsonify({\"error\": error_message}), 500\n\n#This is the step function for the simple UCT algorithm\n# @app.route(\"/api/step\", methods=[\"POST\"])\n# def step():\n# global main_idea, current_node, current_state\n# if current_node is None:\n# return jsonify({\"error\": \"Please enter an initial research idea first\"}), 400\n\n# data = request.get_json()\n# if not data or \"action\" not in data:\n# return jsonify({\"error\": \"Invalid payload\"}), 400\n\n# action = data[\"action\"]\n# use_mcts = data.get('use_mcts', False)\n\n# try:\n# # Handle the automatic generation action using UCT algorithm\n# if action == \"generate\":\n# # Automatically select an action using UCT algorithm\n# # First get valid actions\n# valid_actions = [\"review_and_refine\", \"retrieve_and_refine\", \"refresh_idea\"]\n \n# # If use_mcts is True, log that we're using the MCTS algorithm\n# if use_mcts:\n# chat_messages.append({\n# \"role\": \"system\", \n# \"content\": \"Using MCTS algorithm for automated exploration...\"\n# })\n# # Simple UCT action selection\n# def uct_score(action):\n# # Get the child node for this action if it exists\n# child = None\n# for c in current_node.children:\n# if c.action == action:\n# child = c\n# break\n \n# # If no child exists for this action, it's unexplored (high potential)\n# if not child:\n# return float('inf')\n \n# # Calculate UCT score with exploration constant 1.414 (standard)\n# exploitation = child.value # Use existing value property \n# exploration = 1.414 * math.sqrt(math.log(current_node.visits) / max(child.visits, 1))\n \n# # Add domain-specific heuristics\n# bonus = 0\n# # If low novelty score, prefer retrieve_and_refine\n# if action == \"retrieve_and_refine\" and hasattr(current_node.state, \"review_scores\"):\n# if current_node.state.review_scores.get(\"novelty\", 10) < 7:\n# bonus += 0.2\n \n# # If iteration count is high, prefer refresh_idea occasionally\n# if action == \"refresh_idea\" and current_node.depth > 3:\n# bonus += 0.15\n \n# return exploitation + exploration + bonus\n \n# # Select action with highest UCT score\n# selected_action = max(valid_actions, key=uct_score)\n \n# # Add system message about the selected action\n# chat_messages.append({\n# \"role\": \"system\", \n# \"content\": f\"Auto-generating using '{selected_action.replace('_', ' ')}' strategy...\"\n# })\n \n# # Recursively call this function with the selected action\n# return step_action(selected_action)\n \n# # Add handler for the judge action - needed by review_and_refine\n# elif action == \"judge\":\n# # Use the review agent to get a unified review of the current idea\n# review_data = review_agent.unified_review(current_node.state.current_idea)\n \n# # Add the review scores to the current node's state\n# if review_data:\n# if not hasattr(current_node.state, \"review_scores\") or not current_node.state.review_scores:\n# current_node.state.review_scores = {}\n# if \"scores\" in review_data:\n# current_node.state.review_scores = review_data[\"scores\"]\n# if \"reviews\" in review_data:\n# current_node.state.review_feedback = review_data[\"reviews\"]\n# if \"average_score\" in review_data:\n# current_node.state.average_score = review_data[\"average_score\"]\n \n# # Add system message with review summary\n# avg_score = review_data.get(\"average_score\", 0)\n# chat_messages.append({\n# \"role\": \"system\", \n# \"content\": f\"Review complete. Overall score: {avg_score:.1f}/10\"\n# })\n \n# # Return the review data\n# return jsonify({\n# \"idea\": main_idea,\n# \"nodeId\": current_node.id,\n# \"action\": action,\n# \"depth\": current_node.state.depth,\n# \"review_scores\": review_data.get(\"scores\", {}),\n# \"average_score\": review_data.get(\"average_score\", 0.0),\n# \"review_feedback\": review_data.get(\"reviews\", {})\n# })\n \n# # Handle regular actions with their existing implementation\n# elif action == \"review_and_refine\":\n# # First get unified review\n# review_data = review_agent.unified_review(current_node.state.current_idea)\n \n# # Sort aspects by score to find lowest scoring ones\n# aspect_scores = []\n# if \"scores\" in review_data:\n# aspect_scores = sorted(\n# review_data[\"scores\"].items(),\n# key=lambda x: x[1]\n# )[:3] # Get 3 lowest scoring aspects\n \n# # Get detailed reviews for lowest aspects\n# detailed_reviews = []\n# for aspect, score in aspect_scores:\n# review = structured_review_agent.review_aspect(\n# current_node.state.current_idea,\n# aspect\n# )\n# if review:\n# detailed_reviews.append(review)\n \n# # Create improvement prompt with focused feedback\n# improvement_state = MCTSState(\n# research_goal=current_node.state.research_goal,\n# current_idea=current_node.state.current_idea,\n# retrieved_knowledge=current_node.state.retrieved_knowledge.copy(),\n# feedback=detailed_reviews,\n# depth=current_node.state.depth + 1\n# )\n \n# # Get improved idea from ideation agent\n# response = mcts.ideation_agent.execute_action(\n# \"review_and_refine\",\n# {\n# \"current_idea\": current_node.state.current_idea,\n# \"reviews\": detailed_reviews,\n# \"action_type\": \"execute\"\n# }\n# )\n \n# # Update state with improved idea\n# improvement_state.current_idea = response[\"content\"]\n \n# # Get new review scores\n# new_review = review_agent.unified_review(improvement_state.current_idea)\n# if new_review:\n# improvement_state.review_scores = new_review.get(\"scores\", {})\n# improvement_state.review_feedback = new_review.get(\"reviews\", {})\n# improvement_state.average_score = new_review.get(\"average_score\", 0.0)\n# improvement_state.reward = new_review.get(\"average_score\", 0.0) / 10\n \n# # Create new node and update current\n# new_node = current_node.add_child(improvement_state, action)\n# current_node = new_node\n# main_idea = improvement_state.current_idea\n\n# elif action == \"retrieve_and_refine\":\n# # Generate query based on current idea\n# query_state = {\n# \"current_idea\": current_node.state.current_idea,\n# \"action_type\": \"generate_query\"\n# }\n# query_response = mcts.ideation_agent.execute_action(\"generate_query\", query_state)\n \n# # Add system message about query generation\n# chat_messages.append({\n# \"role\": \"system\",\n# \"content\": \"Generating search query...\"\n# })\n \n# # Get search query\n# query = None\n# try:\n# if isinstance(query_response.get(\"content\"), str):\n# import json\n# # Try to extract query from JSON in content\n# match = re.search(r'{.*}', query_response[\"content\"])\n# if match:\n# query_json = json.loads(match.group())\n# query = query_json.get(\"query\")\n# if not query:\n# # Fallback to using first sentence\n# query = query_response[\"content\"].split(\".\")[0]\n\n# # Add message showing the generated query\n# chat_messages.append({\n# \"role\": \"system\", \n# \"content\": f\"Generated search query: {query}\"\n# })\n\n# except Exception as e:\n# print(f\"Error parsing query: {e}\")\n# query = current_node.state.current_idea[:100] # Fallback to using truncated idea\n# chat_messages.append({\n# \"role\": \"system\",\n# \"content\": f\"Failed to parse query, using fallback: {query}\"\n# })\n \n# # Retrieve relevant papers\n# chat_messages.append({\n# \"role\": \"system\",\n# \"content\": \"Searching for relevant papers...\"\n# })\n \n# search_results = scholar_qa.answer_query(query)\n \n# # Add message showing search results\n# if search_results and \"sections\" in search_results:\n# chat_messages.append({\n# \"role\": \"system\",\n# \"content\": f\"Found {len(search_results['sections'])} relevant sections from papers\"\n# })\n\n# # Create new state with retrieved knowledge\n# retrieval_state = MCTSState(\n# research_goal=current_node.state.research_goal,\n# current_idea=current_node.state.current_idea,\n# retrieved_knowledge=current_node.state.retrieved_knowledge + [search_results],\n# feedback=current_node.state.feedback.copy(),\n# depth=current_node.state.depth + 1\n# )\n \n# # Improve idea with retrieved knowledge\n# improvement_response = mcts.ideation_agent.execute_action(\n# \"retrieve_and_refine\",\n# {\n# \"current_idea\": current_node.state.current_idea,\n# \"retrieved_content\": search_results,\n# \"action_type\": \"execute\"\n# }\n# )\n \n# # Update state with improved idea\n# retrieval_state.current_idea = improvement_response[\"content\"]\n \n# # Get new review scores\n# new_review = review_agent.unified_review(retrieval_state.current_idea)\n# if new_review:\n# retrieval_state.review_scores = new_review.get(\"scores\", {})\n# retrieval_state.review_feedback = new_review.get(\"reviews\", {})\n# retrieval_state.average_score = new_review.get(\"average_score\", 0.0)\n# retrieval_state.reward = new_review.get(\"average_score\", 0.0) / 10\n \n# # Create new node and update current\n# new_node = current_node.add_child(retrieval_state, action)\n# current_node = new_node\n# main_idea = retrieval_state.current_idea\n\n# elif action == \"refresh_idea\":\n# # Get the research goal from the root node\n# research_goal = None\n# if hasattr(current_root.state, \"research_goal\"):\n# research_goal = current_root.state.research_goal\n\n# # Get fresh perspective on the idea\n# response = mcts.ideation_agent.execute_action(\n# \"refresh_idea\",\n# {\n# \"research_goal\": research_goal,\n# \"current_idea\": current_node.state.current_idea,\n# \"action_type\": \"execute\"\n# }\n# )\n \n# # Create a new state with trajectory-level memory\n# # Start with depth 1 since this is directly connected to the root\n# refresh_state = MCTSState(\n# research_goal=research_goal,\n# current_idea=response[\"content\"],\n# retrieved_knowledge=[], # Start with empty retrieved knowledge for new approach\n# feedback={}, # Start with empty feedback for new approach\n# depth=1 # Directly connected to root, so depth is 1\n# )\n \n# # Get new review scores\n# new_review = review_agent.unified_review(refresh_state.current_idea)\n# if new_review:\n# refresh_state.review_scores = new_review.get(\"scores\", {})\n# refresh_state.review_feedback = new_review.get(\"reviews\", {})\n# refresh_state.average_score = new_review.get(\"average_score\", 0.0)\n# refresh_state.reward = new_review.get(\"average_score\", 0.0) / 10\n \n# # Create new node and add as child of the ROOT node instead of current node\n# new_node = current_root.add_child(refresh_state, action)\n \n# # Update current node to the newly created node\n# current_node = new_node\n# main_idea = refresh_state.current_idea\n\n# # Add system message about the refresh\n# chat_messages.append({\n# \"role\": \"system\", \n# \"content\": \"Created a new approach based on the original research goal.\"\n# })\n\n# else:\n# return jsonify({\"error\": \"Invalid action\"}), 400\n\n# # Return updated state\n# return jsonify({\n# \"idea\": main_idea,\n# \"nodeId\": current_node.id,\n# \"action\": action,\n# \"depth\": current_node.state.depth,\n# \"review_scores\": getattr(current_node.state, \"review_scores\", {}),\n# \"average_score\": getattr(current_node.state, \"average_score\", 0.0),\n# \"retrieved_knowledge\": bool(current_node.state.retrieved_knowledge),\n# \"has_feedback\": bool(current_node.state.feedback)\n# })\n\n# except Exception as e:\n# error_message = f\"Error executing {action}: {str(e)}\"\n# traceback.print_exc()\n# return jsonify({\"error\": error_message}), 500\n\n@app.route(\"/api/step\", methods=[\"POST\"])\ndef step():\n global main_idea, current_node, current_state, exploration_in_progress, current_root\n \n if current_node is None:\n return jsonify({\"error\": \"Please enter an initial research idea first\"}), 400\n\n data = request.get_json()\n if not data or \"action\" not in data:\n return jsonify({\"error\": \"Invalid payload\"}), 400\n\n action = data[\"action\"]\n use_mcts = data.get('use_mcts', False)\n num_iterations = data.get('num_iterations', 1)\n max_iterations = data.get('max_iterations', 5)\n\n try:\n # Enhanced MCTS implementation following Algorithm 1 from the PDF\n if action == \"generate\":\n # Prevent concurrent MCTS executions\n if exploration_in_progress:\n return jsonify({\"error\": \"MCTS exploration already in progress\"}), 429\n \n exploration_in_progress = True\n \n try:\n \n # Perform MCTS iterations\n best_node = current_node\n if use_mcts and num_iterations <= max_iterations:\n \n # Phase 1: SELECT - Traverse tree using UCT to find leaf node\n selected_node = mcts_select(current_root)\n \n # Phase 2: EVALUATE - Get reward for the selected state\n reward = mcts_evaluate(selected_node)\n \n # Phase 3: EXPAND - Create children if not terminal and below max depth\n if selected_node.state.depth < mcts.config[\"experiment\"][\"max_depth\"]:\n mcts_expand(selected_node)\n \n # Phase 4: BACKPROPAGATE - Update Q and N values up the tree\n mcts_backpropagate(selected_node, reward)\n \n # Track the best node found so far\n if reward > (getattr(best_node.state, 'average_score', 0) / 10.0):\n best_node = selected_node\n \n # Select the best child of root after all iterations\n final_best = mcts_best_child(current_root)\n if final_best and hasattr(final_best.state, 'average_score'):\n if final_best.state.average_score > (getattr(best_node.state, 'average_score', 0)):\n best_node = final_best\n \n # Update current state to the best found\n current_node = best_node\n main_idea = best_node.state.current_idea\n \n chat_messages.append({\n \"role\": \"system\", \n \"content\": f\"✅ MCTS completed. Best score: {getattr(best_node.state, 'average_score', 0):.1f}/10\"\n })\n \n return jsonify({\n \"idea\": main_idea,\n \"nodeId\": current_node.id,\n \"action\": \"mcts_exploration\",\n \"depth\": current_node.state.depth,\n \"visits\": current_node.visits,\n \"value\": current_node.value,\n \"review_scores\": getattr(current_node.state, \"review_scores\", {}),\n \"average_score\": getattr(current_node.state, \"average_score\", 0.0),\n \"messages\": chat_messages[-5:] # Return last 5 messages\n })\n \n finally:\n exploration_in_progress = False\n \n # Add handler for the judge action - needed by review_and_refine\n elif action == \"judge\":\n # Use the review agent to get a unified review of the current idea\n review_data = review_agent.unified_review(current_node.state.current_idea)\n \n # Add the review scores to the current node's state\n if review_data:\n if not hasattr(current_node.state, \"review_scores\") or not current_node.state.review_scores:\n current_node.state.review_scores = {}\n if \"scores\" in review_data:\n current_node.state.review_scores = review_data[\"scores\"]\n if \"reviews\" in review_data:\n current_node.state.review_feedback = review_data[\"reviews\"]\n if \"average_score\" in review_data:\n current_node.state.average_score = review_data[\"average_score\"]\n \n # Add system message with review summary\n avg_score = review_data.get(\"average_score\", 0)\n chat_messages.append({\n \"role\": \"system\", \n \"content\": f\"Review complete. Overall score: {avg_score:.1f}/10\"\n })\n \n # Return the review data\n return jsonify({\n \"idea\": main_idea,\n \"nodeId\": current_node.id,\n \"action\": action,\n \"depth\": current_node.state.depth,\n \"review_scores\": review_data.get(\"scores\", {}),\n \"average_score\": review_data.get(\"average_score\", 0.0),\n \"review_feedback\": review_data.get(\"reviews\", {})\n })\n \n # Handle regular actions with their existing implementation\n elif action == \"review_and_refine\":\n # First get unified review\n review_data = review_agent.unified_review(current_node.state.current_idea)\n \n # Sort aspects by score to find lowest scoring ones\n aspect_scores = []\n if \"scores\" in review_data:\n aspect_scores = sorted(\n review_data[\"scores\"].items(),\n key=lambda x: x[1]\n )[:3] # Get 3 lowest scoring aspects\n \n # Get detailed reviews for lowest aspects\n detailed_reviews = []\n for aspect, score in aspect_scores:\n review = structured_review_agent.review_aspect(\n current_node.state.current_idea,\n aspect\n )\n if review:\n detailed_reviews.append(review)\n \n # Create improvement prompt with focused feedback\n improvement_state = MCTSState(\n research_goal=current_node.state.research_goal,\n current_idea=current_node.state.current_idea,\n retrieved_knowledge=current_node.state.retrieved_knowledge.copy(),\n feedback=detailed_reviews,\n depth=current_node.state.depth + 1\n )\n \n # Get improved idea from ideation agent\n response = mcts.ideation_agent.execute_action(\n \"review_and_refine\",\n {\n \"current_idea\": current_node.state.current_idea,\n \"reviews\": detailed_reviews,\n \"action_type\": \"execute\"\n }\n )\n \n # Update state with improved idea\n improvement_state.current_idea = response[\"content\"]\n \n # Get new review scores\n new_review = review_agent.unified_review(improvement_state.current_idea)\n if new_review:\n improvement_state.review_scores = new_review.get(\"scores\", {})\n improvement_state.review_feedback = new_review.get(\"reviews\", {})\n improvement_state.average_score = new_review.get(\"average_score\", 0.0)\n improvement_state.reward = new_review.get(\"average_score\", 0.0) / 10\n \n # Create new node and update current\n new_node = current_node.add_child(improvement_state, action)\n current_node = new_node\n main_idea = improvement_state.current_idea\n\n # IMPLEMENT MISSING retrieve_and_refine ACTION\n elif action == \"retrieve_and_refine\":\n # Step 1: Generate search query based on current idea\n chat_messages.append({\n \"role\": \"system\",\n \"content\": \"Generating search query for knowledge retrieval...\"\n })\n \n # Generate query using ideation agent (consistent with frontend)\n query_response = mcts.ideation_agent.execute_action(\n \"generate_query\",\n {\n \"current_idea\": current_node.state.current_idea,\n \"action_type\": \"generate_query\"\n }\n )\n \n # Extract query from response\n query = None\n try:\n content = query_response.get(\"content\", \"\")\n # Try to parse JSON response first\n json_match = re.search(r'{.*}', content)\n if json_match:\n query_json = json.loads(json_match.group())\n query = query_json.get(\"query\", content.split(\".\")[0])\n else:\n # Fallback to using first sentence\n query = content.split(\".\")[0] if content else current_node.state.current_idea[:100]\n \n chat_messages.append({\n \"role\": \"system\", \n \"content\": f\"Generated search query: {query}\"\n })\n \n except Exception as e:\n print(f\"Error parsing query: {e}\")\n query = current_node.state.current_idea[:100] # Fallback\n \n # Step 2: Retrieve relevant knowledge using ScholarQA\n chat_messages.append({\n \"role\": \"system\",\n \"content\": \"Searching for relevant papers...\"\n })\n \n try:\n search_results = scholar_qa.answer_query(query)\n \n if search_results and \"sections\" in search_results:\n chat_messages.append({\n \"role\": \"system\",\n \"content\": f\"Found {len(search_results['sections'])} relevant sections from papers\"\n })\n else:\n search_results = {\"sections\": [], \"query\": query}\n chat_messages.append({\n \"role\": \"system\",\n \"content\": \"No relevant papers found, proceeding without additional knowledge\"\n })\n \n except Exception as e:\n print(f\"Error in knowledge retrieval: {e}\")\n search_results = {\"sections\": [], \"query\": query}\n chat_messages.append({\n \"role\": \"system\",\n \"content\": f\"Error in retrieval: {str(e)}\"\n })\n \n # Step 3: Improve idea with retrieved knowledge\n chat_messages.append({\n \"role\": \"system\",\n \"content\": \"Refining idea with retrieved knowledge...\"\n })\n \n # Create new state with retrieved knowledge\n retrieval_state = MCTSState(\n research_goal=current_node.state.research_goal,\n current_idea=current_node.state.current_idea,\n retrieved_knowledge=current_node.state.retrieved_knowledge + [search_results],\n feedback=current_node.state.feedback.copy(),\n depth=current_node.state.depth + 1\n )\n \n # Improve idea with retrieved knowledge using ideation agent\n improvement_response = mcts.ideation_agent.execute_action(\n \"retrieve_and_refine\",\n {\n \"current_idea\": current_node.state.current_idea,\n \"retrieved_content\": search_results,\n \"action_type\": \"execute\"\n }\n )\n \n # Update state with improved idea\n retrieval_state.current_idea = improvement_response[\"content\"]\n \n # Step 4: Get new review scores for the improved idea\n new_review = review_agent.unified_review(retrieval_state.current_idea)\n if new_review:\n retrieval_state.review_scores = new_review.get(\"scores\", {})\n retrieval_state.review_feedback = new_review.get(\"reviews\", {})\n retrieval_state.average_score = new_review.get(\"average_score\", 0.0)\n retrieval_state.reward = new_review.get(\"average_score\", 0.0) / 10\n \n # Create new node and update current\n new_node = current_node.add_child(retrieval_state, action)\n current_node = new_node\n main_idea = retrieval_state.current_idea\n \n chat_messages.append({\n \"role\": \"system\",\n \"content\": f\"Idea refined with retrieved knowledge. New score: {getattr(retrieval_state, 'average_score', 0):.1f}/10\"\n })\n\n elif action == \"refresh_idea\":\n # Get the research goal from the root node\n research_goal = None\n if hasattr(current_root.state, \"research_goal\"):\n research_goal = current_root.state.research_goal\n\n # Get fresh perspective on the idea\n response = mcts.ideation_agent.execute_action(\n \"refresh_idea\",\n {\n \"research_goal\": research_goal,\n \"current_idea\": current_node.state.current_idea,\n \"action_type\": \"execute\"\n }\n )\n \n # Create a new state with trajectory-level memory\n # # Start with depth 1 since this is directly connected to the root\n # parent_depth = current_node.parent.state.depth if current_node.parent else 0\n if response and \"content\" in response:\n refresh_state = MCTSState(\n research_goal=research_goal,\n current_idea=response[\"content\"],\n retrieved_knowledge=[], # Start with empty retrieved knowledge for new approach\n feedback={}, # Start with empty feedback for new approach\n depth=current_node.state.depth +1 # Directly connected to root, so depth is 1\n )\n \n # Get new review scores\n new_review = review_agent.unified_review(refresh_state.current_idea)\n if new_review:\n refresh_state.review_scores = new_review.get(\"scores\", {})\n refresh_state.review_feedback = new_review.get(\"reviews\", {})\n refresh_state.average_score = new_review.get(\"average_score\", 0.0)\n refresh_state.reward = new_review.get(\"average_score\", 0.0) / 10\n \n # FIXED: Create new node as child of current node's PARENT (sibling relationship)\n if current_node.parent is not None:\n # Current node has a parent - create sibling\n parent_node = current_node.parent\n new_node = parent_node.add_child(refresh_state, action)\n else:\n # Current node IS the root - create child of root\n new_node = current_root.add_child(refresh_state, action)\n \n # Update current node to the newly created node\n current_node = new_node\n main_idea = refresh_state.current_idea\n\n # Add system message about the refresh\n chat_messages.append({\n \"role\": \"system\", \n \"content\": \"Created a new approach based on the original research goal.\"\n })\n\n else:\n return jsonify({\"error\": \"Invalid action\"}), 400\n\n # Return updated state\n return jsonify({\n \"idea\": main_idea,\n \"nodeId\": current_node.id,\n \"action\": action,\n \"depth\": current_node.state.depth,\n \"review_scores\": getattr(current_node.state, \"review_scores\", {}),\n \"average_score\": getattr(current_node.state, \"average_score\", 0.0),\n \"retrieved_knowledge\": bool(current_node.state.retrieved_knowledge),\n \"has_feedback\": bool(current_node.state.feedback)\n })\n\n except Exception as e:\n exploration_in_progress = False\n error_message = f\"Error executing {action}: {str(e)}\"\n traceback.print_exc()\n return jsonify({\"error\": error_message}), 500\n\n\ndef execute_mcts_action(state, action):\n \"\"\"Execute an action within MCTS to create a new state\"\"\"\n try:\n if action == \"review_and_refine\":\n # Get current reviews if not available\n if not hasattr(state, \"review_scores\") or not state.review_scores:\n review_data = review_agent.unified_review(state.current_idea)\n if review_data:\n state.review_scores = review_data.get(\"scores\", {})\n state.review_feedback = review_data.get(\"reviews\", {})\n state.average_score = review_data.get(\"average_score\", 0.0)\n \n # Find lowest scoring aspects for improvement\n if hasattr(state, \"review_scores\") and state.review_scores:\n sorted_aspects = sorted(state.review_scores.items(), key=lambda x: x[1])[:3]\n detailed_reviews = []\n for aspect, score in sorted_aspects:\n if hasattr(state, \"review_feedback\") and aspect in state.review_feedback:\n detailed_reviews.append({\n \"aspect\": aspect,\n \"score\": score,\n \"feedback\": state.review_feedback[aspect]\n })\n else:\n detailed_reviews = []\n \n # Get improved idea\n response = mcts.ideation_agent.execute_action(\n \"review_and_refine\",\n {\n \"current_idea\": state.current_idea,\n \"reviews\": detailed_reviews,\n \"action_type\": \"execute\"\n }\n )\n \n # Create new state\n new_state = MCTSState(\n research_goal=state.research_goal,\n current_idea=response[\"content\"],\n retrieved_knowledge=state.retrieved_knowledge.copy(),\n feedback=state.feedback.copy(),\n depth=state.depth + 1\n )\n \n return new_state\n \n elif action == \"retrieve_and_refine\":\n # FIXED: Implement proper retrieve_and_refine logic\n # Step 1: Generate search query\n query_response = mcts.ideation_agent.execute_action(\n \"generate_query\",\n {\n \"current_idea\": state.current_idea,\n \"action_type\": \"generate_query\"\n }\n )\n \n # Extract query from response\n query = None\n try:\n content = query_response.get(\"content\", \"\")\n json_match = re.search(r'{.*}', content)\n if json_match:\n query_json = json.loads(json_match.group())\n query = query_json.get(\"query\", content.split(\".\")[0])\n else:\n query = content.split(\".\")[0] if content else state.current_idea[:100]\n except Exception as e:\n logger.error(f\"Error parsing query: {e}\")\n query = state.current_idea[:100]\n \n # Step 2: Retrieve knowledge\n try:\n search_results = scholar_qa.answer_query(query)\n if not search_results or \"sections\" not in search_results:\n search_results = {\"sections\": [], \"query\": query}\n except Exception as e:\n logger.error(f\"Error in knowledge retrieval: {e}\")\n search_results = {\"sections\": [], \"query\": query}\n \n # Step 3: Improve idea with retrieved knowledge\n if search_results.get(\"sections\"):\n improvement_response = mcts.ideation_agent.execute_action(\n \"retrieve_and_refine\",\n {\n \"current_idea\": state.current_idea,\n \"retrieved_content\": search_results,\n \"action_type\": \"execute\"\n }\n )\n \n new_state = MCTSState(\n research_goal=state.research_goal,\n current_idea=improvement_response[\"content\"],\n retrieved_knowledge=state.retrieved_knowledge + [search_results],\n feedback=state.feedback.copy(),\n depth=state.depth + 1\n )\n \n return new_state\n else:\n # No knowledge retrieved, return None to indicate failed action\n return None\n \n elif action == \"refresh_idea\":\n # Generate fresh approach\n response = mcts.ideation_agent.execute_action(\n \"refresh_idea\",\n {\n \"research_goal\": state.research_goal,\n \"current_idea\": state.current_idea,\n \"action_type\": \"execute\"\n }\n )\n \n new_state = MCTSState(\n research_goal=state.research_goal,\n current_idea=response[\"content\"],\n retrieved_knowledge=[], # Fresh start\n feedback={},\n depth=1 # Connect to root level\n )\n \n return new_state\n \n except Exception as e:\n logger.error(f\"Error executing MCTS action {action}: {e}\")\n return None\n\n\ndef mcts_expand(node):\n \"\"\"Phase 2: EXPAND - Add new child nodes for unexplored actions\"\"\"\n if node.state.depth >= mcts.config[\"experiment\"][\"max_depth\"]:\n return\n \n valid_actions = [\"review_and_refine\", \"retrieve_and_refine\", \"refresh_idea\"]\n \n for action in valid_actions:\n if not any(child.action == action for child in node.children):\n try:\n # Execute action to create new state\n new_state = execute_mcts_action(node.state, action)\n \n if new_state:\n # # SPECIAL HANDLING FOR REFRESH_IDEA - create sibling instead of child\n # if action == \"refresh_idea\":\n # # Add as child of parent (sibling of current node)\n # parent_node = node.parent if node.parent else node # Fallback to current if no parent\n # child = parent_node.add_child(new_state, action)\n # else:\n # # Regular child creation for other actions\n child = node.add_child(new_state, action)\n child.state.depth = node.state.depth + 1\n \n # Evaluate the new child/sibling node\n try:\n review_data = review_agent.unified_review(new_state.current_idea)\n if review_data:\n child.state.review_scores = review_data.get(\"scores\", {})\n child.state.review_feedback = review_data.get(\"reviews\", {})\n child.state.average_score = review_data.get(\"average_score\", 0.0)\n child.state.reward = review_data.get(\"average_score\", 0.0) / 10\n \n logger.info(f\"Child node {child.id} evaluated with score: {child.state.average_score}\")\n else:\n child.state.average_score = 5.0\n child.state.reward = 0.5\n \n except Exception as review_error:\n logger.error(f\"Error evaluating child node {child.id}: {review_error}\")\n child.state.average_score = 5.0\n child.state.reward = 0.5\n \n logger.info(f\"Expanded node {node.id} with action {action}, created {'sibling' if action == 'refresh_idea' else 'child'} {child.id}\")\n \n except Exception as e:\n logger.error(f\"Error expanding with action {action}: {e}\")\n\n\ndef mcts_select(root_node):\n \"\"\"Phase 1: SELECT - Traverse tree using UCT to find leaf node\"\"\"\n current = root_node\n path = [current]\n \n while True:\n # If node has unvisited children, select one randomly\n unvisited_children = [child for child in current.children if child.visits == 0]\n if unvisited_children:\n selected = random.choice(unvisited_children)\n path.append(selected)\n return selected\n \n # If all children visited, use UCT to select next node\n if not current.children:\n return current # Leaf node\n \n # Calculate UCT scores for all children\n best_child = None\n best_uct = float('-inf')\n \n for child in current.children:\n if child.visits == 0:\n uct_value = float('inf') # Unvisited nodes have infinite priority\n else:\n # UCT formula from PDF: Q(n)/N(n) + c * sqrt(ln(N(parent))/N(n))\n exploitation = child.value # Q(n)/N(n) is already calculated in child.value\n exploration = math.sqrt(math.log(max(current.visits, 1)) / child.visits)\n uct_value = exploitation + (1.414 * exploration) # c = sqrt(2) ≈ 1.414\n \n if uct_value > best_uct:\n best_uct = uct_value\n best_child = child\n \n if best_child is None:\n return current\n \n current = best_child\n path.append(current)\n \n # If we've reached a leaf or terminal node, return it\n if not current.children or current.state.depth >= mcts.config[\"experiment\"][\"max_depth\"]:\n return current\n\n\ndef mcts_evaluate(node):\n \"\"\"Phase 2: EVALUATE - Get reward for the selected state via Review Agent\"\"\"\n try:\n # If node already has review scores, use them\n if hasattr(node.state, 'average_score') and node.state.average_score > 0:\n return node.state.average_score / 10.0 # Normalize to 0-1\n \n # Otherwise, get fresh review from Review Agent\n review_data = review_agent.unified_review(node.state.current_idea)\n \n if review_data and \"average_score\" in review_data:\n # Update node state with review data\n node.state.review_scores = review_data.get(\"scores\", {})\n node.state.review_feedback = review_data.get(\"reviews\", {})\n node.state.average_score = review_data[\"average_score\"]\n node.state.reward = review_data[\"average_score\"] / 10.0\n \n return node.state.reward\n \n # Fallback reward based on depth (deeper = less reward)\n return 1.0 / (node.state.depth + 1)\n \n except Exception as e:\n logger.error(f\"Error in MCTS evaluate: {e}\")\n return 0.5 # Default middle reward\n\n\n\ndef mcts_backpropagate(node, reward):\n \"\"\"Phase 4: BACKPROPAGATE - Update Q and N values for node and ancestors\"\"\"\n current = node\n current_reward = reward\n \n while current is not None:\n # Update visit count\n current.visits += 1\n \n # Update value using incremental average: value += (reward - value) / visits\n current.value += (current_reward - current.value) / current.visits\n \n # Apply discount factor for parent nodes\n current_reward *= mcts.discount_factor\n \n # Move to parent\n current = current.parent\n\n\ndef mcts_best_child(node):\n \"\"\"Select the child with highest average reward Q/N\"\"\"\n if not node.children:\n return None\n \n best_child = None\n best_value = float('-inf')\n \n for child in node.children:\n if child.visits > 0 and child.value > best_value:\n best_value = child.value\n best_child = child\n \n return best_child\n\n# Helper function to avoid code duplication\ndef step_action(action):\n \"\"\"Execute a step with a specific action and return the result\"\"\"\n # We simulate a new request with the selected action\n request_data = {\"action\": action}\n \n # Create a mock Request object\n class MockRequest:\n def get_json(self):\n return request_data\n \n # Store the original request\n original_request = request\n \n # Set the global request to our mock\n globals()[\"request\"] = MockRequest()\n \n try:\n # Call the step function with our mock request\n result = step()\n return result\n finally:\n # Restore the original request\n globals()[\"request\"] = original_request\n\n@app.route(\"/api/tree\", methods=[\"GET\"])\ndef get_tree():\n if current_root is None:\n return jsonify({}), 200 # Return empty object instead of error\n\n def node_to_dict(node):\n # Get basic info about the node\n is_root = node.parent is None\n \n # For root node, use special formatting to show it's the research goal\n if is_root:\n node_data = {\n \"id\": node.id,\n \"action\": \"research_goal\", # Special action type for root\n \"idea\": \"RESEARCH GOAL: \" + (node.state.research_goal[:80] + \"...\" if len(node.state.research_goal) > 80 else node.state.research_goal),\n \"depth\": node.state.depth,\n \"reward\": node.state.reward,\n \"value\": node.value,\n \"visits\": node.visits,\n \"isCurrentNode\": node.id == current_node.id if current_node else False,\n \"state\": {\n \"current_idea\": node.state.research_goal, # For root, use research_goal as current_idea\n \"depth\": node.state.depth,\n \"reward\": node.state.reward,\n \"hasReviews\": False, # Root node has no reviews\n \"hasRetrieval\": False, # Root node has no retrieval\n \"hasFeedback\": False, # Root node has no feedback\n \"isResearchGoal\": True # Flag to identify it's the research goal\n },\n \"children\": [node_to_dict(child) for child in node.children]\n }\n else:\n # Regular node formatting for ideas\n node_data = {\n \"id\": node.id,\n \"action\": node.action or \"unknown\",\n \"idea\": node.state.current_idea[:100] + \"...\" if len(node.state.current_idea) > 100 else node.state.current_idea,\n \"depth\": node.state.depth,\n \"reward\": node.state.reward,\n \"value\": node.value,\n \"visits\": node.visits,\n \"isCurrentNode\": node.id == current_node.id if current_node else False,\n \"state\": {\n \"current_idea\": node.state.current_idea,\n \"depth\": node.state.depth,\n \"reward\": node.state.reward,\n \"hasReviews\": hasattr(node.state, \"review_scores\") and bool(node.state.review_scores),\n \"hasRetrieval\": bool(node.state.retrieved_knowledge),\n \"hasFeedback\": bool(node.state.feedback),\n \"isResearchGoal\": False # Regular nodes are not research goals\n },\n \"children\": [node_to_dict(child) for child in node.children]\n }\n \n # Add review data if available\n if hasattr(node.state, \"review_scores\") and node.state.review_scores:\n node_data[\"reviews\"] = {\n \"scores\": node.state.review_scores,\n \"summary\": getattr(node.state, \"review_summary\", {})\n }\n \n return node_data\n\n tree_data = node_to_dict(current_root)\n return jsonify(tree_data)\n\n\n@app.route(\"/api/node\", methods=[\"POST\"])\ndef select_node():\n global current_node, main_idea, chat_messages\n data = request.get_json()\n if not data or \"node_id\" not in data:\n return jsonify({\"error\": \"Invalid payload\"}), 400\n\n node_id = data[\"node_id\"]\n \n def find_node(root, target_id):\n \"\"\"Recursively find a node by ID in the tree.\"\"\"\n if root.id == target_id:\n return root\n for child in root.children:\n result = find_node(child, target_id)\n if result:\n return result\n return None\n\n # Find the node in the tree\n node = find_node(current_root, node_id)\n \n if node:\n # Update current node and idea\n current_node = node\n main_idea = node.state.current_idea\n \n # Log this action in chat\n chat_messages.append({\n \"role\": \"system\", \n \"content\": f\"Navigated to node {node_id} with action '{node.action or 'root'}'.\"\n })\n \n # Prepare response data\n response = {\n \"idea\": main_idea,\n \"node_data\": {\n \"id\": node.id,\n \"action\": node.action or \"root\",\n \"depth\": node.state.depth,\n \"reward\": node.state.reward\n },\n \"review_scores\": getattr(node.state, \"review_scores\", {}),\n \"average_score\": getattr(node.state, \"average_score\", 0.0),\n }\n \n # Include review data if available\n if hasattr(node.state, \"review_scores\") and node.state.review_scores:\n response[\"reviews\"] = {\n \"scores\": node.state.review_scores,\n \"feedback\": node.state.review_feedback,\n \"summary\": getattr(node.state, \"review_summary\", {})\n }\n \n # Include trajectory history\n if node.parent:\n trajectory = []\n current = node\n while current.parent:\n trajectory.append({\n \"id\": current.id,\n \"action\": current.action,\n \"depth\": current.state.depth\n })\n current = current.parent\n response[\"trajectory\"] = list(reversed(trajectory))\n \n return jsonify(response)\n else:\n return jsonify({\"error\": f\"Node with ID {node_id} not found\"}), 404\n\n\n@app.route(\"/api/idea\", methods=[\"GET\"])\ndef get_idea():\n if current_node is None:\n return jsonify({\"idea\": main_idea})\n \n return jsonify({\n \"idea\": main_idea,\n \"review_scores\": getattr(current_node.state, \"review_scores\", {}),\n \"average_score\": getattr(current_node.state, \"average_score\", 0.0),\n })\n\n\n@app.route(\"/api/upload\", methods=[\"POST\"])\ndef upload_file():\n if \"file\" not in request.files:\n return jsonify({\"error\": \"No file part\"}), 400\n\n file = request.files[\"file\"]\n if file.filename == \"\":\n return jsonify({\"error\": \"No selected file\"}), 400\n\n # Initialize abstract variable\n abstract = \"Abstract could not be extracted\" # Default value in case no abstract is found\n\n if file and allowed_file(file.filename):\n filename = secure_filename(file.filename)\n unique_filename = f\"{uuid.uuid4()}_{filename}\"\n file_path = os.path.join(app.config[\"UPLOAD_FOLDER\"], unique_filename)\n\n try:\n file.save(file_path)\n file_content = \"File content could not be extracted\"\n \n # Extract text content from PDF using PyMuPDF if it's a PDF\n if file_path.lower().endswith('.pdf'):\n try:\n pdf_text = []\n with pymupdf.open(file_path) as doc:\n for page_num in range(len(doc)):\n page = doc.load_page(page_num)\n pdf_text.append(page.get_text())\n file_content = \"\\n\".join(pdf_text)\n # Extract the abstract from the PDF content\n abstract = extract_abstract(pdf_text)\n except Exception as pdf_err:\n print(f\"Error extracting PDF content: {pdf_err}\")\n \n # Add file info to knowledge chunks\n new_id = len(knowledge_chunks) + 1\n chunk = {\n \"id\": new_id,\n \"text\": f\"Uploaded file: {filename}\",\n \"full_text\": file_content,\n \"abstract\": abstract, # Added abstract here\n \"source\": file_path,\n \"file_type\": \"attachment\",\n }\n knowledge_chunks.append(chunk)\n\n return (\n jsonify(\n {\n \"message\": \"File uploaded successfully\",\n \"filename\": filename,\n \"abstract\": abstract,\n \"chunk\": chunk,\n }\n ),\n 201,\n )\n\n except Exception as e:\n return jsonify({\"error\": f\"Error saving file: {str(e)}\"}), 500\n\n return jsonify({\"error\": \"File type not allowed\"}), 400\n\n\n@app.route(\"/api/review_aspect\", methods=[\"POST\"])\ndef review_aspect():\n data = request.get_json()\n if not data or \"idea\" not in data or \"aspect\" not in data:\n return jsonify({\"error\": \"Invalid payload\"}), 400\n\n idea = data[\"idea\"]\n aspect = data[\"aspect\"]\n aspect_index = data.get(\"aspect_index\", 0)\n\n try:\n print(f\"\\n=== REQUEST DATA ===\")\n print(f\"Aspect: {aspect}\")\n print(f\"Aspect Index: {aspect_index}\")\n print(f\"Idea length: {len(idea)} chars\")\n \n review_data = structured_review_agent.review_aspect(idea, aspect)\n \n print(f\"\\n=== RESPONSE DATA ===\")\n print(f\"Review Data Keys: {review_data.keys()}\")\n if 'highlight' in review_data:\n print(f\"Highlight Data Keys: {review_data['highlight'].keys()}\")\n print(f\"Highlight Text: '{review_data['highlight'].get('text')}'\")\n \n # Add debugging to check if highlight text exists in idea\n highlight_text = review_data['highlight'].get('text')\n if highlight_text:\n # Check for exact match\n exact_match = highlight_text in idea\n print(f\"Exact match found in idea: {exact_match}\")\n \n # Check for case-insensitive match\n case_insensitive_match = highlight_text.lower() in idea.lower()\n print(f\"Case-insensitive match found in idea: {case_insensitive_match}\")\n \n # Check if text might be affected by markdown formatting\n if not exact_match:\n # Try to find position where highlight might be with markdown formatting\n stripped_highlight = highlight_text.replace('*', '').replace('_', '')\n stripped_match = stripped_highlight in idea\n print(f\"Match after removing markdown chars: {stripped_match}\")\n \n # Additional check: Look for parts of the highlight text\n if len(highlight_text) > 15:\n partial_text = highlight_text[5:15]\n partial_match = partial_text in idea\n print(f\"Partial match with '{partial_text}': {partial_match}\")\n \n response = {\"review_data\": review_data, \"aspect_index\": aspect_index}\n print(f\"\\n=== SENDING RESPONSE ===\")\n print(f\"Response Keys: {response.keys()}\")\n \n return jsonify(response)\n except Exception as e:\n error_message = f\"Error reviewing {aspect}: {str(e)}\"\n print(f\"\\n=== ERROR ===\")\n print(error_message)\n return jsonify({\"error\": error_message}), 500\n\n\n@app.route(\"/api/improve_idea\", methods=[\"POST\"])\ndef improve_idea():\n global main_idea, current_node\n data = request.get_json()\n if not data or \"idea\" not in data or \"accepted_reviews\" not in data:\n return jsonify({\"error\": \"Invalid payload\"}), 400\n\n idea = data[\"idea\"]\n accepted_reviews = data[\"accepted_reviews\"]\n\n print(\"Accepted reviews:\", accepted_reviews)\n # print(\"Idea:\", idea)\n\n try:\n # Use ideation_agent instead of structured_review_agent\n improved_idea = ideation_agent.improve_idea(idea, accepted_reviews)\n\n # Update the main idea in our application state\n main_idea = improved_idea\n\n # Create a new state with trajectory-level memory\n new_state = MCTSState(\n research_goal=current_node.state.research_goal,\n current_idea=improved_idea,\n retrieved_knowledge=current_node.state.retrieved_knowledge.copy(),\n feedback=current_node.state.feedback.copy(),\n depth=current_node.state.depth + 1\n )\n \n # Get review using the unified review method\n review_data = review_agent.unified_review(improved_idea)\n print(f\"Review score: {review_data['average_score']}\")\n if review_data:\n if \"scores\" in review_data:\n new_state.review_scores = review_data[\"scores\"]\n if \"reviews\" in review_data:\n new_state.review_feedback = review_data[\"reviews\"]\n if \"average_score\" in review_data:\n new_state.average_score = review_data[\"average_score\"]\n new_state.reward = review_data[\"average_score\"]\n \n # Create new node and add as child of current node\n new_node = current_node.add_child(new_state, \"review_and_refine\")\n current_node = new_node\n\n # Add a system message about the improvement\n chat_messages.append(\n {\n \"role\": \"system\",\n \"content\": \"Idea improved based on accepted review suggestions.\",\n }\n )\n\n # Add the improved idea as an assistant message\n chat_messages.append({\"role\": \"assistant\", \"content\": improved_idea})\n\n return jsonify({\n \"improved_idea\": improved_idea,\n \"review_scores\": getattr(current_node.state, \"review_scores\", {}),\n \"average_score\": getattr(current_node.state, \"average_score\", 0.0),\n })\n \n except Exception as e:\n error_message = f\"Error improving idea: {str(e)}\"\n return jsonify({\"error\": error_message}), 500\n\n\n@app.route(\"/api/generate_query\", methods=[\"POST\"])\ndef generate_query():\n \"\"\"Generate a research query based on the current idea\"\"\"\n data = request.get_json()\n \n if not data or \"idea\" not in data:\n return jsonify({\"error\": \"Missing idea in request\"}), 400\n \n idea = data[\"idea\"]\n \n try:\n # Log the attempt in chat\n global chat_messages\n chat_messages.append({\n \"role\": \"system\",\n \"content\": \"Generating search query based on your research idea...\"\n })\n \n # Use the ideation agent to generate a query\n prompt = \"Given the following research idea, generate a concise and specific query for retrieving relevant scientific papers. The query should be focused on the key aspects of the research idea. Format your response as a JSON object with a single field named 'query'.\"\n \n # Call the ideation agent\n response = ideation_agent.execute_action(\n \"generate_query\",\n {\"current_idea\": idea, \"prompt\": prompt}\n )\n \n # Extract the query from the response\n content = response.get(\"content\", \"\")\n \n # Try to parse JSON response\n try:\n # Look for JSON in the response\n json_match = re.search(r'```json\\s*(.*?)\\s*```|{.*}', content, re.DOTALL)\n if json_match:\n json_str = json_match.group(1) or json_match.group(0)\n query_data = json.loads(json_str)\n query = query_data.get(\"query\", \"\")\n else:\n # Try to extract query directly\n query_match = re.search(r'query[\"\\']?\\s*:\\s*[\"\\']([^\"\\']+)[\"\\']', content)\n if query_match:\n query = query_match.group(1)\n else:\n # Fallback - use the first sentence as query\n query = content.split(\".\")[0]\n except Exception as e:\n # If JSON parsing fails, use a simple text extraction approach\n lines = content.split(\"\\n\")\n for line in lines:\n if \"query\" in line.lower():\n query = line.split(\":\", 1)[1].strip().strip('\"\\'')\n break\n else:\n query = content.strip()\n \n # Add the generated query to chat\n chat_messages.append({\n \"role\": \"assistant\",\n \"content\": f\"**Generated search query:** \\\"{query}\\\"\\n\\nI'll use this query to find relevant papers. You can click 'Retrieve Knowledge' to proceed with this query.\"\n })\n \n return jsonify({\"query\": query})\n \n except Exception as e:\n error_trace = traceback.format_exc()\n print(f\"Query generation error: {str(e)}\\n{error_trace}\")\n chat_messages.append({\n \"role\": \"system\",\n \"content\": f\"Error generating query: {str(e)}\"\n })\n return jsonify({\"error\": f\"Failed to generate query: {str(e)}\"}), 500\n\n@app.route(\"/api/retrieve_knowledge\", methods=[\"POST\"])\ndef retrieve_knowledge():\n \"\"\"Retrieve knowledge based on a query\"\"\"\n data = request.get_json()\n \n if not data or \"query\" not in data:\n return jsonify({\"error\": \"Missing query in request\"}), 400\n \n query = data[\"query\"]\n \n try:\n # Log the attempt in chat\n global chat_messages\n chat_messages.append({\n \"role\": \"system\",\n \"content\": f\"Searching for relevant papers using query: \\\"{query}\\\"...\"\n })\n\n print(f\"Retrieving knowledge for query: {query}\")\n \n # Use ScholarQA to retrieve knowledge\n result = scholar_qa.answer_query(query)\n \n # Store the retrieval results globally\n global retrieval_results\n retrieval_results = result\n \n # Parse the sections and format for display\n formatted_sections = []\n \n for section in result.get(\"sections\", []):\n formatted_section = {\n \"title\": section.get(\"title\", \"Untitled Section\"),\n \"summary\": section.get(\"tldr\", \"No summary available\"),\n \"content\": section.get(\"text\", \"\"),\n \"citations\": []\n }\n \n # Extract citations\n for citation in section.get(\"citations\", []):\n citation_id = citation.get(\"id\", \"\")\n paper = citation.get(\"paper\", {})\n \n formatted_citation = {\n \"id\": citation_id,\n \"title\": paper.get(\"title\", \"Unknown paper\"),\n \"authors\": [author.get(\"name\", \"\") for author in paper.get(\"authors\", [])],\n \"year\": paper.get(\"year\", \"\"),\n \"venue\": paper.get(\"venue\", \"\"),\n \"url\": f\"https://api.semanticscholar.org/CorpusID:{paper.get('corpus_id', '')}\"\n }\n \n formatted_section[\"citations\"].append(formatted_citation)\n \n formatted_sections.append(formatted_section)\n \n # Add success message to chat\n sections_count = len(formatted_sections)\n citations_count = sum(len(section[\"citations\"]) for section in formatted_sections)\n \n chat_messages.append({\n \"role\": \"assistant\",\n \"content\": f\"✅ **Retrieval complete!** Found {sections_count} content sections with {citations_count} paper citations.\\n\\nPlease check the left panel to see the retrieved information.\"\n })\n \n return jsonify({\n \"query\": query,\n \"sections\": formatted_sections\n })\n \n except Exception as e:\n error_trace = traceback.format_exc()\n print(f\"Retrieval error: {str(e)}\\n{error_trace}\")\n chat_messages.append({\n \"role\": \"system\",\n \"content\": f\"Error retrieving knowledge: {str(e)}\"\n })\n return jsonify({\"error\": f\"Failed to retrieve knowledge: {str(e)}\"}), 500\n\n@app.route(\"/api/improve_idea_with_knowledge\", methods=[\"POST\"])\ndef improve_idea_with_knowledge():\n \"\"\"Improve the research idea based on retrieved knowledge.\"\"\"\n global main_idea, current_node, retrieval_results\n \n data = request.get_json()\n if not data or \"idea\" not in data:\n return jsonify({\"error\": \"Missing idea in request\"}), 400\n \n idea = data[\"idea\"]\n \n # Check if we have any retrieval results to use\n if not retrieval_results or \"sections\" not in retrieval_results:\n return jsonify({\"error\": \"No retrieved knowledge available\"}), 400\n \n try:\n # Format the retrieved knowledge for the ideation agent\n retrieved_content = []\n for section in retrieval_results.get(\"sections\", []):\n section_text = f\"## {section.get('title', 'Untitled')}\\n\\n\"\n section_text += f\"{section.get('text', '')}\\n\\n\"\n \n # Add citations if available\n if section.get(\"citations\"):\n section_text += \"### References:\\n\"\n for citation in section.get(\"citations\", []):\n paper = citation.get(\"paper\", {})\n authors = \", \".join([author.get(\"name\", \"\") for author in paper.get(\"authors\", [])[:3]])\n if len(paper.get(\"authors\", [])) > 3:\n authors += \" et al.\"\n section_text += f\"- {paper.get('title', 'Untitled')} ({authors}, {paper.get('year', 'n.d.')})\\n\"\n \n retrieved_content.append(section_text)\n \n # Join all sections into a single text\n formatted_knowledge = \"\\n\\n\".join(retrieved_content)\n \n # Add system message about the improvement process\n chat_messages.append({\n \"role\": \"system\",\n \"content\": \"Improving research idea with retrieved knowledge...\"\n })\n \n # Add instructions to ensure markdown format and making only relevant improvements\n prompt_instructions = {\n \"current_idea\": idea,\n \"retrieved_content\": formatted_knowledge,\n }\n \n # Call the ideation agent to improve the idea based on the retrieved knowledge\n response = ideation_agent.execute_action(\n \"refine_with_retrieval\",\n prompt_instructions\n )\n \n # Extract the improved idea, handling potential JSON format\n content = response.get(\"content\", \"\")\n improved_idea = content\n \n # Try to extract content from JSON if present\n try:\n # Look for JSON in response\n import re\n import json\n \n # Check if the response contains JSON\n json_match = re.search(r'```json\\s*(.*?)\\s*```|{.*}', content, re.DOTALL)\n if json_match:\n json_str = json_match.group(1) or json_match.group(0)\n # Clean up the extracted JSON string\n json_str = json_str.replace('\\\\n', ' ').strip()\n \n # Parse the JSON\n parsed_json = json.loads(json_str)\n if \"content\" in parsed_json:\n improved_idea = parsed_json[\"content\"]\n elif \"text\" in parsed_json:\n improved_idea = parsed_json[\"text\"]\n # If no content/text key but other text field exists, use that\n else:\n for key in [\"body\", \"description\", \"idea\"]:\n if key in parsed_json:\n improved_idea = parsed_json[key]\n break\n except Exception as json_error:\n print(f\"JSON extraction error (continuing with raw content): {str(json_error)}\")\n # If JSON parsing fails, keep the original content\n pass\n \n # Ensure improved_idea isn't empty\n if not improved_idea or improved_idea.strip() == \"\":\n improved_idea = content\n \n print(f\"Improved idea length: {len(improved_idea)} characters\")\n \n # Update the main idea in our application state\n main_idea = improved_idea\n \n # Create a new state with trajectory-level memory\n new_state = MCTSState(\n research_goal=current_node.state.research_goal,\n current_idea=improved_idea,\n retrieved_knowledge=current_node.state.retrieved_knowledge.copy() + [retrieval_results.get(\"query\", \"\")],\n feedback=current_node.state.feedback.copy(),\n depth=current_node.state.depth + 1\n )\n \n # Get review using the unified review method\n review_data = review_agent.unified_review(improved_idea)\n print(f\"Review score: {review_data['average_score']}\")\n if review_data:\n if \"scores\" in review_data:\n new_state.review_scores = review_data[\"scores\"]\n if \"reviews\" in review_data:\n new_state.review_feedback = review_data[\"reviews\"]\n if \"average_score\" in review_data:\n new_state.average_score = review_data[\"average_score\"]\n new_state.reward = review_data[\"average_score\"]\n \n # Create new node and add as child of current node\n new_node = current_node.add_child(new_state, \"retrieve_and_refine\")\n current_node = new_node\n \n # Add a system message about the improvement\n chat_messages.append({\n \"role\": \"system\",\n \"content\": \"Idea improved based on retrieved knowledge.\"\n })\n \n # Add the improved idea as an assistant message\n chat_messages.append({\"role\": \"assistant\", \"content\": improved_idea})\n \n return jsonify({\n \"improved_idea\": improved_idea,\n \"content_length\": len(improved_idea),\n \"review_scores\": getattr(new_state, \"review_scores\", {}),\n \"average_score\": getattr(new_state, \"average_score\", 0.0),\n })\n \n except Exception as e:\n error_message = f\"Error improving idea with knowledge: {str(e)}\"\n error_trace = traceback.format_exc()\n print(f\"{error_message}\\n{error_trace}\")\n return jsonify({\"error\": error_message}), 500\n\n@app.route(\"/api/refresh_idea\", methods=[\"POST\"])\ndef refresh_idea():\n \"\"\"Dedicated endpoint for refreshing research ideas\"\"\"\n global main_idea, current_node, chat_messages, current_root\n \n # Check if current_node exists\n if current_node is None:\n return jsonify({\"error\": \"No active research idea found. Please start by entering a research topic.\"}), 400\n \n try:\n # Add system message before refresh\n chat_messages.append(\n {\"role\": \"system\", \"content\": \"Generating a completely new approach to the research goal...\"}\n )\n \n # Get the research goal from the root node\n research_goal = None\n if hasattr(current_root.state, \"research_goal\"):\n research_goal = current_root.state.research_goal\n \n # Call the ideation agent to get a refreshed idea\n response = mcts.ideation_agent.execute_action(\n \"refresh_idea\", \n {\n \"research_goal\": research_goal,\n \"current_idea\": main_idea, \n \"action_type\": \"execute\"\n }\n )\n \n # Extract the idea content from the response\n new_idea = response.get(\"content\", \"\")\n \n # Create a new state with trajectory-level memory\n # Start with depth 1 since this is directly connected to the root\n new_state = MCTSState(\n research_goal=research_goal,\n current_idea=new_idea,\n retrieved_knowledge=[], # Start with empty retrieved knowledge for new approach\n feedback={}, # Start with empty feedback for new approach\n depth=1 # Directly connected to root, so depth is 1\n )\n \n # Get review using the unified review method\n review_data = review_agent.unified_review(new_idea)\n print(f\"Review score: {review_data['average_score']}\")\n if review_data:\n if \"scores\" in review_data:\n new_state.review_scores = review_data[\"scores\"]\n if \"reviews\" in review_data:\n new_state.review_feedback = review_data[\"reviews\"]\n if \"average_score\" in review_data:\n new_state.average_score = review_data[\"average_score\"]\n new_state.reward = review_data[\"average_score\"]\n \n # Create new node and add as child of the ROOT node instead of current node\n new_node = current_root.add_child(new_state, \"refresh_idea\")\n \n # Update current node to the newly created node\n current_node = new_node\n \n # Update main idea\n main_idea = new_idea\n\n # Add completion message to chat\n chat_messages.append({\"role\": \"system\", \"content\": \"Created a new approach based on the original research goal.\"})\n\n return jsonify({\n \"idea\": new_idea,\n \"messages\": chat_messages,\n \"review_scores\": getattr(new_state, \"review_scores\", {}),\n \"average_score\": getattr(new_state, \"average_score\", 0.0),\n })\n \n except Exception as e:\n error_message = f\"Error refreshing idea: {str(e)}\"\n chat_messages.append({\"role\": \"system\", \"content\": error_message})\n return jsonify({\"error\": error_message, \"messages\": chat_messages}), 500\n\n# WebSocket endpoints for real-time MCTS exploration\n@socketio.on('start_exploration')\ndef handle_start_exploration():\n global exploration_in_progress, current_state\n \n if exploration_in_progress:\n return {'error': 'Exploration already in progress'}\n \n exploration_in_progress = True\n \n try:\n # Create initial state if none exists\n if not current_state:\n current_state = MCTSState()\n \n # Start MCTS exploration with callback for updates\n def exploration_callback(message):\n emit('exploration_update', {\n 'type': 'progress',\n 'message': message\n })\n \n # Run MCTS with callback\n root = mcts.run(current_state, num_iterations=5, callback=exploration_callback)\n \n # Store best path\n best_node = root\n while best_node.children:\n best_node = max(best_node.children, key=lambda n: n.value)\n \n # Update current state\n current_state = best_node.state\n \n # Send final results\n emit('exploration_complete', {\n 'idea': current_state.current_idea,\n 'score': current_state.average_score if hasattr(current_state, 'average_score') else None,\n 'tree_data': root.to_dict()\n })\n \n except Exception as e:\n emit('exploration_error', {'error': str(e)})\n finally:\n exploration_in_progress = False\n\n@socketio.on('stop_exploration')\ndef handle_stop_exploration():\n global exploration_in_progress\n exploration_in_progress = False\n emit('exploration_stopped')\n\n@app.route(\"/api/set_aspect_weights\", methods=[\"POST\"])\ndef set_aspect_weights():\n \"\"\"Update the weights for different review aspects.\"\"\"\n try:\n data = request.get_json()\n if not data or \"weights\" not in data:\n return jsonify({\"error\": \"Missing weights\"}), 400\n \n weights = data[\"weights\"]\n # Validate that we have all required aspects\n required_aspects = [\"novelty\", \"clarity\", \"feasibility\", \"effectiveness\", \"impact\"]\n if not all(aspect in weights for aspect in required_aspects):\n return jsonify({\"error\": \"Missing required aspects\"}), 400\n \n # Update weights in review agent\n review_agent.set_aspect_weights(weights)\n \n return jsonify({\n \"success\": True,\n \"message\": \"Aspect weights updated successfully\"\n })\n \n except Exception as e:\n error_message = f\"Error setting aspect weights: {str(e)}\"\n print(error_message)\n return jsonify({\"error\": error_message}), 500\n\n@app.cli.command(\"check-config\")\ndef check_configuration():\n \"\"\"Check the application configuration and environment.\"\"\"\n issues = []\n warnings = []\n \n # Check Flask configuration\n if not os.environ.get('FLASK_SECRET_KEY'):\n if os.environ.get('FLASK_ENV') == 'production':\n issues.append(\"FLASK_SECRET_KEY must be set in production\")\n else:\n warnings.append(\"FLASK_SECRET_KEY not set - using default (unsafe for production)\")\n \n # Check required API keys\n required_keys = {\n 'GEMINI_API_KEY': 'Google Gemini',\n 'HUGGINGFACE_API_KEY': 'HuggingFace',\n 'SEMANTIC_SCHOLAR_API_KEY': 'Semantic Scholar'\n }\n \n for env_var, provider in required_keys.items():\n if not os.environ.get(env_var):\n issues.append(f\"Missing required {provider} API key ({env_var})\")\n \n # Check optional API keys\n optional_keys = {\n 'OPENAI_API_KEY': 'OpenAI',\n 'ANTHROPIC_API_KEY': 'Anthropic/Claude',\n 'DEEPSEEK_API_KEY': 'DeepSeek'\n }\n \n for env_var, provider in optional_keys.items():\n if not os.environ.get(env_var):\n warnings.append(f\"Optional {provider} API key not configured ({env_var})\")\n \n # Check secure directories\n if not os.path.exists(SECURE_KEYS_DIR):\n issues.append(f\"Secure keys directory {SECURE_KEYS_DIR} does not exist\")\n elif os.environ.get('FLASK_ENV') == 'production':\n # Check directory permissions in production\n import stat\n key_dir_stat = os.stat(SECURE_KEYS_DIR)\n if key_dir_stat.st_mode & stat.S_IROTH or key_dir_stat.st_mode & stat.S_IWOTH:\n issues.append(f\"Secure keys directory {SECURE_KEYS_DIR} has unsafe permissions\")\n \n # Check upload directory\n if not os.path.exists(app.config[\"UPLOAD_FOLDER\"]):\n issues.append(f\"Upload directory {app.config['UPLOAD_FOLDER']} does not exist\")\n \n # Print results\n if issues:\n click.secho(\"\\n❌ Configuration issues found:\", fg='red', bold=True)\n for issue in issues:\n click.secho(f\" - {issue}\", fg='red')\n \n if warnings:\n click.secho(\"\\n⚠️ Configuration warnings:\", fg='yellow', bold=True)\n for warning in warnings:\n click.secho(f\" - {warning}\", fg='yellow')\n \n if not issues and not warnings:\n click.secho(\"\\n✅ Configuration valid!\", fg='green', bold=True)\n elif not issues:\n click.secho(\"\\n✅ No critical issues found (see warnings above)\", fg='green')\n \n return len(issues) == 0\n\nif __name__ == \"__main__\":\n socketio.run(app, debug=True)\n"}}}} |