1 line
391 KiB
JSON
1 line
391 KiB
JSON
{"repo": "answers111/alpha-research", "files": {"benchmark/minizing_raio_max_min_distance/initial_program.py": {"3d2c029a76234620e3b67b26f3f1f9f4a2f4787c": {"t": 1785599210.7983546, "n": 8893, "h": "7d8e1276188c6f1f", "c": "import numpy as np\nfrom scipy.spatial.distance import pdist\nfrom scipy.spatial import cKDTree\n\n# (Removed) smooth_points — smoothing logic is now inlined to reduce indirection\n\n\ndef calculate_distances(points):\n \"\"\"Calculates min, max, and ratio of pairwise Euclidean distances using scipy pdist.\"\"\"\n if points.shape[0] < 2:\n return 0.0, 0.0, 0.0\n distances = pdist(points, metric='euclidean')\n eps = 1e-8\n min_dist = max(np.min(distances), eps)\n max_dist = np.max(distances)\n ratio = max_dist / min_dist\n return min_dist, max_dist, ratio\n\n# (Removed) perturb_point — now inlined directly where used\n\ndef update_temperature(temperature, cooling_rate, accept_history, iteration, total_iters, initial_temperature, window_size=100):\n \"\"\"\n Adaptive cooling with acceptance‐rate feedback and periodic reheating.\n \"\"\"\n window = accept_history[-min(len(accept_history), window_size):]\n rate = sum(window) / len(window)\n # gentler correction: slow/fast cooling factors reduced\n if rate < 0.2:\n adj = 1.02\n elif rate > 0.8:\n adj = 0.98\n else:\n adj = 1.0\n temperature *= cooling_rate * adj\n # removed periodic reheating to maintain smoother cooling schedule\n # if (iteration + 1) % (total_iters // 4) == 0:\n # temperature = initial_temperature\n return temperature\n\ndef max_min_dis_ratio(n: int, d: int, seed=None):\n \"\"\"\n Finds n points in d-dimensional space to minimize the max/min distance ratio \n using simulated annealing.\n\n Args:\n n (int): Number of points.\n d (int): Dimensionality of the space.\n\n Returns:\n tuple: (best_points, best_ratio)\n \"\"\"\n\n # Adaptive hyperparameters based on dimensionality\n iterations = 3000 if d <= 2 else 6000 # increased sweeps for improved convergence\n initial_temperature = 10.0\n cooling_rate = 0.998 if d <= 2 else 0.996 # slower cooling for extended exploration\n perturbation_factor = 0.15 if d <= 2 else 0.12 # tuned smaller steps in 3D for better local refinement\n # relaxation factor for post-acceptance repulsive adjustment\n # relaxation_factor removed; using inline 0.1 * perturbation_factor below\n\n # 1. Initial State: reproducible random generator\n rng = np.random.default_rng(seed)\n # uniform random initialization in [0,1]^d for simplicity\n current_points = rng.random((n, d))\n \n _, _, current_ratio = calculate_distances(current_points)\n \n best_points = np.copy(current_points)\n best_ratio = current_ratio\n \n temperature = initial_temperature\n accept_history = []\n window_size = 50 # window for stagnation detection and adaptive injection\n # smoothing_interval remains, but smoothing_strength is fixed inlined above\n smoothing_interval = max(10, iterations // (20 if d <= 2 else 30)) # more frequent smoothing in 3D for improved uniformity\n \n for i in range(iterations):\n # Build KD-tree once per iteration for neighbor queries\n tree = cKDTree(current_points)\n # optional smoothing step using distance-weighted neighbor smoothing\n if (i + 1) % smoothing_interval == 0:\n # choose neighbor count based on dimension\n k_smooth = 6 if d > 2 else 4\n _, idxs = tree.query(current_points, k=k_smooth+1)\n neighbors = current_points[idxs[:,1:]] # exclude self\n # compute inverse-distance weights\n diffs = neighbors - current_points[:, None, :]\n dists = np.linalg.norm(diffs, axis=2) + 1e-6\n weights = 1.0 / dists\n weights /= weights.sum(axis=1, keepdims=True)\n neighbor_means = (neighbors * weights[..., None]).sum(axis=1)\n blend = 0.6 if d > 2 else 0.7\n current_points = np.clip(current_points * blend + neighbor_means * (1 - blend), 0.0, 1.0)\n _, _, current_ratio = calculate_distances(current_points)\n if current_ratio < best_ratio:\n best_points = current_points.copy()\n best_ratio = current_ratio\n\n # 2. Generate Neighboring State: Perturb a random point\n # Simplify scaling: rely on temperature to adjust step-size instead of best_ratio\n # dynamic perturbation decays sublinearly with temperature for finer local moves\n perturbation_strength = perturbation_factor * ((temperature / initial_temperature)**0.6 + 0.15)\n \n # Choose a random point to perturb\n point_to_perturb_index = rng.integers(0, n)\n \n old_point = current_points[point_to_perturb_index].copy()\n # Increase repulsive‐move frequency in low dimensions\n # dynamic repulsion probability: stronger at high temperature, tapering off as we cool\n if d > 2:\n # reduce repulsion frequency in 3D for finer refinement\n repulsion_prob = float(np.clip(temperature / initial_temperature, 0.2, 0.8))\n else:\n repulsion_prob = float(np.clip(temperature / initial_temperature + 0.1, 0.5, 0.95))\n # start with a random jitter\n # random jitter inlined for readability\n candidate = old_point + rng.uniform(-perturbation_strength, perturbation_strength, size=old_point.shape)\n if n > 1 and rng.random() < repulsion_prob:\n # compute nearest neighbor via KD-tree for efficiency (reusing prebuilt tree)\n _, nn_idxs = tree.query(old_point, k=2)\n nn_idx = nn_idxs[1]\n vec = old_point - current_points[nn_idx]\n norm = np.linalg.norm(vec)\n if norm > 1e-8:\n dir_vec = vec / norm\n candidate = old_point + perturbation_strength * dir_vec\n # keep the point in [0,1]^d\n current_points[point_to_perturb_index] = np.clip(candidate, 0.0, 1.0)\n _, _, candidate_ratio = calculate_distances(current_points)\n \n # Acceptance criterion\n delta = candidate_ratio - current_ratio\n accept = (delta < 0) or (rng.random() < np.exp(-delta / temperature))\n\n if accept:\n current_ratio = candidate_ratio\n # Post-acceptance repulsive relaxation to improve local spacing\n # reuse prebuilt KD-tree for repulsive relaxation\n dists, idxs_nn = tree.query(current_points[point_to_perturb_index], k=2)\n dir_vec = current_points[point_to_perturb_index] - current_points[idxs_nn[1]]\n norm = np.linalg.norm(dir_vec)\n if norm > 1e-8:\n # push away from nearest neighbor\n adjustment = 0.1 * perturbation_factor * dir_vec / norm\n current_points[point_to_perturb_index] = np.clip(\n current_points[point_to_perturb_index] + adjustment, 0.0, 1.0\n )\n # update ratio and best points after relaxation\n _, _, relaxed_ratio = calculate_distances(current_points)\n current_ratio = relaxed_ratio\n if relaxed_ratio < best_ratio:\n best_points = current_points.copy()\n best_ratio = relaxed_ratio\n # also keep the standard best‐check for the candidate move\n if current_ratio < best_ratio:\n best_points = current_points.copy()\n best_ratio = current_ratio\n else:\n current_points[point_to_perturb_index] = old_point\n \n # Update temperature with adaptive schedule\n accept_history.append(accept)\n temperature = update_temperature(temperature, cooling_rate, accept_history, i, iterations, initial_temperature)\n # periodic mild reheating for 3D to escape deep minima\n if d > 2 and (i + 1) % (iterations // 3) == 0:\n temperature = max(temperature, initial_temperature * 0.3)\n\n # random injection to escape plateaus: reinitialize one point every 20% of iterations\n # random injection only if we’ve stagnated (low acceptance in recent window)\n if (i + 1) % max(1, iterations // 5) == 0 and len(accept_history) >= window_size \\\n and sum(accept_history[-window_size:]) / window_size < 0.1:\n j = rng.integers(0, n)\n current_points[j] = rng.random(d)\n _, _, current_ratio = calculate_distances(current_points)\n\n # Local refinement stage: fine-tune best solution with small Gaussian perturbations\n refine_iters = max(100, iterations // 20)\n for _ in range(refine_iters):\n idx = rng.integers(0, n)\n old_point = best_points[idx].copy()\n perturb = rng.normal(0, perturbation_factor * 0.05, size=d)\n best_points[idx] = np.clip(old_point + perturb, 0.0, 1.0)\n _, _, refined_ratio = calculate_distances(best_points)\n if refined_ratio < best_ratio:\n best_ratio = refined_ratio\n else:\n best_points[idx] = old_point\n return best_points, best_ratio"}}, "benchmark/heilbronn_in_the_unit_square/evaluator.py": {"77092e56d045b09d8d321235a5e937de79852689": {"t": 1785599210.8132985, "n": 2856, "h": "4775b84db4f1458a", "c": "import numpy as np\nimport importlib.util\nimport sys\nimport os\nimport json\nfrom itertools import combinations\nfrom typing import Dict\n\n\ndef _triangle_area(a, b, c) -> float:\n return abs((b[0]-a[0])*(c[1]-a[1]) - (b[1]-a[1])*(c[0]-a[0])) * 0.5\n\n\ndef evaluate_min_triangle_area(points: np.ndarray) -> Dict[str, float]:\n \"\"\"\n Compute the minimum triangle area for 2D points.\n Metrics:\n - min_area: raw smallest triangle area (larger is better)\n - scaled_min_area: n^(8/7 + 1/2000) * min_area (informational)\n - score: equals min_area (larger is better)\n \"\"\"\n pts = np.asarray(points, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 2:\n return {\"valid\": 0.0, \"min_area\": 0.0, \"n\": 0.0, \"scaled_min_area\": 0.0, \"score\": 0.0}\n\n n = len(pts)\n if n < 3:\n return {\"valid\": 0.0, \"min_area\": 0.0, \"n\": float(n), \"scaled_min_area\": 0.0, \"score\": 0.0}\n\n min_area = float(\"inf\")\n for i, j, k in combinations(range(n), 3):\n area = _triangle_area(pts[i], pts[j], pts[k])\n if area < min_area:\n min_area = area\n if min_area <= 1e-18: # 早停\n break\n\n exponent = (8.0/7.0) + (1.0/2000.0)\n scaled_min_area = (n ** exponent) * float(min_area)\n score = float(min_area) if np.isfinite(min_area) else 0.0\n\n return {\n \"valid\": 1.0,\n \"min_area\": float(min_area),\n \"n\": float(n),\n \"scaled_min_area\": float(scaled_min_area),\n \"score\": score,\n }\n\n\n\ndef evaluate(program_path: str):\n try:\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n\n points = None\n if hasattr(program, 'points'):\n points = program.points\n elif hasattr(program, 'main'):\n res = program.main()\n if isinstance(res, np.ndarray):\n points = res\n elif hasattr(program, 'points'):\n points = program.points\n if points is None:\n return {\"error\": -1.0}\n result = evaluate_min_triangle_area(points)\n # Return both the diagnostic dict and rely on 'score' as the main metric (larger is better)\n return {\n \"score\": result[\"score\"],\n \"scaled_min_area\": result[\"scaled_min_area\"],\n \"min_area\": result[\"min_area\"],\n \"n\": result[\"n\"],\n }\n except Exception:\n return {\"error\": -1.0}\n\n\nif __name__ == \"__main__\":\n try:\n default_path = os.path.join(os.path.dirname(__file__), \"initial_program.py\")\n except Exception:\n default_path = \"initial_program.py\"\n target = sys.argv[1] if len(sys.argv) > 1 else default_path\n print(json.dumps(evaluate(target), ensure_ascii=False, indent=2))\n"}}, "benchmark/minizing_raio_max_min_distance/evaluator.py": {"9179134e613a2d93a4922e8573bfdb0ad10edfa7": {"t": 1785599210.86035, "n": 1658, "h": "cd01f9169964884e", "c": "import numpy as np\nimport scipy as sp\n\n\ndef cal_ratio(construction_1):\n pairwise_distances = sp.spatial.distance.pdist(construction_1)\n min_distance = np.min(pairwise_distances)\n max_distance = np.max(pairwise_distances)\n ratio_squared = (min_distance / max_distance)**2\n return ratio_squared\n\n\ndef evaluate(program_path: str = \"results/initial_program.py\"):\n \"\"\"\n Evaluate the pack_circles function from the given program file.\n Returns the total radius sum if valid, otherwise raises an exception.\n \"\"\"\n import importlib.util\n import sys\n \n # Load the module from the given path\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n\n\n # Check if 'max_min_dis_ratio' exists in the loaded module\n if not hasattr(program, 'max_min_dis_ratio'):\n raise ValueError(f\"The file '{program_path}' does not define 'max_min_dis_ratio'.\")\n\n try:\n res_n16_d2, _ = program.max_min_dis_ratio(16, 2) \n res_n14_d3, _ = program.max_min_dis_ratio(14, 3)\n except Exception as e1:\n return {\"result\": -10.0, \"error\": e1}\n \n try:\n ratio_n16_d2 = cal_ratio(res_n16_d2) # AlphaEvolve: 1 / 12.88926611203463 = 0.07758393622320406\n ratio_n14_d3 = cal_ratio(res_n14_d3) # AlphaEvolve: 1 / 4.165849767 = 0.24004706263470807\n\n\n except Exception as e:\n return {\"result\": -1.0, \"error\": e}\n\n results = {\n \"ratio_n16_d2\": ratio_n16_d2,\n \"ratio_n14_d3\": ratio_n14_d3, \n }\n \n return results\n\nprint(evaluate())"}}, "configs/default_config.yaml": {"52eadc4f7cc5ca9a19ab1601a7ae6d8d58735a56": {"t": 1785599210.8703752, "n": 6612, "h": "499f610e695c5257", "c": "# EvolveAgent Default Configuration\n# This file contains all available configuration options with sensible defaults\n# You can use this as a template for your own configuration\n\n# General settings\nmax_iterations: 1000 # Maximum number of evolution iterations\ncheckpoint_interval: 50 # Save checkpoints every N iterations\nlog_level: \"INFO\" # Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)\nlog_dir: null # Custom directory for logs (default: output_dir/logs)\nrandom_seed: null # Random seed for reproducibility (null = random)\n\n# Evolution settings\ndiff_based_evolution: true # Use diff-based evolution (true) or full rewrites (false)\nallow_full_rewrites: false # Allow occasional full rewrites even in diff-based mode\nmax_code_length: 100000 # Maximum allowed code length in characters\n\n# LLM configuration\nllm:\n # Models for evolution\n models:\n # List of available models with their weights\n - name: \"deepseek-chat\"\n weight: 1.0\n # - name: \"gemini-2.0-flash\"\n # weight: 0\n\n # Models for LLM feedback\n # evaluator_models:\n # # List of available models with their weights\n # - name: \"gemini-2.0-flash-lite\"\n # weight: 0.8\n # - name: \"gemini-2.0-flash\"\n # weight: 0.2\n\n # API configuration\n api_base: \"https://api.deepseek.com\" # Base URL for API (change for non-OpenAI models)\n api_key: \"sk-2c3f1f58031b4b86afdb6a8192ea02e2\" # API key (defaults to OPENAI_API_KEY env variable)\n\n # Generation parameters\n temperature: 0.7 # Temperature for generation (higher = more creative)\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 8192 # Maximum tokens to generate\n\n # Request parameters\n timeout: 300 # Timeout for API requests in seconds\n retries: 3 # Number of retries for failed requests\n retry_delay: 5 # Delay between retries in seconds\n\n# Prompt configuration\nprompt:\n template_dir: null # Custom directory for prompt templates\n # system_message: \"You are an expert coder helping to improve programs through evolution.\"\n # evaluator_system_message: \"You are an expert code reviewer.\"\n\n # Number of examples to include in the prompt\n num_top_programs: 3 # Number of top-performing programs to include\n num_diverse_programs: 2 # Number of diverse programs to include\n\n # Template stochasticity\n use_template_stochasticity: true # Use random variations in templates for diversity\n template_variations: # Different phrasings for parts of the template\n improvement_suggestion:\n - \"Here's how we could improve this code:\"\n - \"I suggest the following improvements:\"\n - \"We can enhance this code by:\"\n\n # Note: meta-prompting features are not yet implemented\n\n# Database configuration\ndatabase:\n # General settings\n db_path: null # Path to persist database (null = in-memory only)\n in_memory: true # Keep database in memory for faster access\n log_prompts: true # If true, log all prompts and responses into the database\n\n # Evolutionary parameters\n population_size: 1000 # Maximum number of programs to keep in memory\n archive_size: 100 # Size of elite archive\n num_islands: 5 # Number of islands for island model (separate populations)\n\n # Island-based evolution parameters\n # Islands provide diversity by maintaining separate populations that evolve independently.\n # Migration periodically shares the best solutions between adjacent islands.\n migration_interval: 50 # Migrate between islands every N generations\n migration_rate: 0.1 # Fraction of top programs to migrate (0.1 = 10%)\n\n # Selection parameters\n elite_selection_ratio: 0.1 # Ratio of elite programs to select\n exploration_ratio: 0.2 # Ratio of exploration vs exploitation\n exploitation_ratio: 0.7 # Ratio of exploitation vs random selection\n # Note: diversity_metric is fixed to \"edit_distance\" (feature_based not implemented)\n\n # Feature map dimensions for MAP-Elites\n feature_dimensions: # Dimensions for MAP-Elites feature map\n - \"score\" # Performance score\n - \"complexity\" # Code complexity (length)\n feature_bins: 10 # Number of bins per dimension\n\n# Evaluator configuration\nevaluator:\n # General settings\n timeout: 300 # Maximum evaluation time in seconds\n max_retries: 3 # Maximum number of retries for evaluation\n\n # Note: resource limits (memory_limit_mb, cpu_limit) are not yet implemented\n\n # Evaluation strategies\n cascade_evaluation: false # Use cascade evaluation to filter bad solutions early\n cascade_thresholds: # Thresholds for advancing to next evaluation stage\n - 0.5 # First stage threshold\n - 0.75 # Second stage threshold\n - 0.9 # Third stage threshold\n\n # Parallel evaluation\n parallel_evaluations: 4 # Number of parallel evaluations\n # Note: distributed evaluation is not yet implemented\n\n # LLM-based feedback (experimental)\n use_llm_feedback: false # Use LLM to evaluate code quality\n llm_feedback_weight: 0.1 # Weight for LLM feedback in final score\n\n# Reward model configuration\nrewardmodel:\n model_type: vllm # Model type (vllm or api)\n model_name: /data/zhuotaodeng/yzj/alpha_research_model/qwen25_grm_iclr_boxed/checkpoint-180 # Model name (if null, uses default)\n temperature: 0.7 # Temperature for generation\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n proposal_score_threshold: 5.5 # Only generate programs if proposal score >= threshold\n # api_key: sk-2c3f1f58031b4b86afdb6a8192ea02e2 # API key for API models\n # base_url: https://api.deepseek.com # Base URL for API models\n jsonl_file: \"results/reward_results.jsonl\" # JSONL file for results\n max_retries: 50 # Maximum number of retries\n retry_delay: 5 # Delay between retries in seconds\n"}, "a23d37cd2d42f059e00ae59ca0996862a2e0e1bf": {"t": 1785611308.9547102, "n": 6612, "h": "499f610e695c5257", "c": "# EvolveAgent Default Configuration\n# This file contains all available configuration options with sensible defaults\n# You can use this as a template for your own configuration\n\n# General settings\nmax_iterations: 1000 # Maximum number of evolution iterations\ncheckpoint_interval: 50 # Save checkpoints every N iterations\nlog_level: \"INFO\" # Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)\nlog_dir: null # Custom directory for logs (default: output_dir/logs)\nrandom_seed: null # Random seed for reproducibility (null = random)\n\n# Evolution settings\ndiff_based_evolution: true # Use diff-based evolution (true) or full rewrites (false)\nallow_full_rewrites: false # Allow occasional full rewrites even in diff-based mode\nmax_code_length: 100000 # Maximum allowed code length in characters\n\n# LLM configuration\nllm:\n # Models for evolution\n models:\n # List of available models with their weights\n - name: \"deepseek-chat\"\n weight: 1.0\n # - name: \"gemini-2.0-flash\"\n # weight: 0\n\n # Models for LLM feedback\n # evaluator_models:\n # # List of available models with their weights\n # - name: \"gemini-2.0-flash-lite\"\n # weight: 0.8\n # - name: \"gemini-2.0-flash\"\n # weight: 0.2\n\n # API configuration\n api_base: \"https://api.deepseek.com\" # Base URL for API (change for non-OpenAI models)\n api_key: \"sk-2c3f1f58031b4b86afdb6a8192ea02e2\" # API key (defaults to OPENAI_API_KEY env variable)\n\n # Generation parameters\n temperature: 0.7 # Temperature for generation (higher = more creative)\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 8192 # Maximum tokens to generate\n\n # Request parameters\n timeout: 300 # Timeout for API requests in seconds\n retries: 3 # Number of retries for failed requests\n retry_delay: 5 # Delay between retries in seconds\n\n# Prompt configuration\nprompt:\n template_dir: null # Custom directory for prompt templates\n # system_message: \"You are an expert coder helping to improve programs through evolution.\"\n # evaluator_system_message: \"You are an expert code reviewer.\"\n\n # Number of examples to include in the prompt\n num_top_programs: 3 # Number of top-performing programs to include\n num_diverse_programs: 2 # Number of diverse programs to include\n\n # Template stochasticity\n use_template_stochasticity: true # Use random variations in templates for diversity\n template_variations: # Different phrasings for parts of the template\n improvement_suggestion:\n - \"Here's how we could improve this code:\"\n - \"I suggest the following improvements:\"\n - \"We can enhance this code by:\"\n\n # Note: meta-prompting features are not yet implemented\n\n# Database configuration\ndatabase:\n # General settings\n db_path: null # Path to persist database (null = in-memory only)\n in_memory: true # Keep database in memory for faster access\n log_prompts: true # If true, log all prompts and responses into the database\n\n # Evolutionary parameters\n population_size: 1000 # Maximum number of programs to keep in memory\n archive_size: 100 # Size of elite archive\n num_islands: 5 # Number of islands for island model (separate populations)\n\n # Island-based evolution parameters\n # Islands provide diversity by maintaining separate populations that evolve independently.\n # Migration periodically shares the best solutions between adjacent islands.\n migration_interval: 50 # Migrate between islands every N generations\n migration_rate: 0.1 # Fraction of top programs to migrate (0.1 = 10%)\n\n # Selection parameters\n elite_selection_ratio: 0.1 # Ratio of elite programs to select\n exploration_ratio: 0.2 # Ratio of exploration vs exploitation\n exploitation_ratio: 0.7 # Ratio of exploitation vs random selection\n # Note: diversity_metric is fixed to \"edit_distance\" (feature_based not implemented)\n\n # Feature map dimensions for MAP-Elites\n feature_dimensions: # Dimensions for MAP-Elites feature map\n - \"score\" # Performance score\n - \"complexity\" # Code complexity (length)\n feature_bins: 10 # Number of bins per dimension\n\n# Evaluator configuration\nevaluator:\n # General settings\n timeout: 300 # Maximum evaluation time in seconds\n max_retries: 3 # Maximum number of retries for evaluation\n\n # Note: resource limits (memory_limit_mb, cpu_limit) are not yet implemented\n\n # Evaluation strategies\n cascade_evaluation: false # Use cascade evaluation to filter bad solutions early\n cascade_thresholds: # Thresholds for advancing to next evaluation stage\n - 0.5 # First stage threshold\n - 0.75 # Second stage threshold\n - 0.9 # Third stage threshold\n\n # Parallel evaluation\n parallel_evaluations: 4 # Number of parallel evaluations\n # Note: distributed evaluation is not yet implemented\n\n # LLM-based feedback (experimental)\n use_llm_feedback: false # Use LLM to evaluate code quality\n llm_feedback_weight: 0.1 # Weight for LLM feedback in final score\n\n# Reward model configuration\nrewardmodel:\n model_type: vllm # Model type (vllm or api)\n model_name: /data/zhuotaodeng/yzj/alpha_research_model/qwen25_grm_iclr_boxed/checkpoint-180 # Model name (if null, uses default)\n temperature: 0.7 # Temperature for generation\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n proposal_score_threshold: 5.5 # Only generate programs if proposal score >= threshold\n # api_key: sk-2c3f1f58031b4b86afdb6a8192ea02e2 # API key for API models\n # base_url: https://api.deepseek.com # Base URL for API models\n jsonl_file: \"results/reward_results.jsonl\" # JSONL file for results\n max_retries: 50 # Maximum number of retries\n retry_delay: 5 # Delay between retries in seconds\n"}, "d1f68c311c4ca77714aac0436a01a222a8501514": {"t": 1785611310.2099445, "n": 6540, "h": "2f783eca7af78045", "c": "# EvolveAgent Default Configuration\n# This file contains all available configuration options with sensible defaults\n# You can use this as a template for your own configuration\n\n# General settings\nmax_iterations: 1000 # Maximum number of evolution iterations\ncheckpoint_interval: 50 # Save checkpoints every N iterations\nlog_level: \"INFO\" # Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)\nlog_dir: null # Custom directory for logs (default: output_dir/logs)\nrandom_seed: null # Random seed for reproducibility (null = random)\n\n# Evolution settings\ndiff_based_evolution: true # Use diff-based evolution (true) or full rewrites (false)\nallow_full_rewrites: false # Allow occasional full rewrites even in diff-based mode\nmax_code_length: 10000 # Maximum allowed code length in characters\n\n# LLM configuration\nllm:\n # Models for evolution\n models:\n # List of available models with their weights\n - name: \"deepseek-chat\"\n weight: 1.0\n # - name: \"gemini-2.0-flash\"\n # weight: 0\n\n # Models for LLM feedback\n # evaluator_models:\n # # List of available models with their weights\n # - name: \"gemini-2.0-flash-lite\"\n # weight: 0.8\n # - name: \"gemini-2.0-flash\"\n # weight: 0.2\n\n # API configuration\n api_base: \"https://api.deepseek.com\" # Base URL for API (change for non-OpenAI models)\n api_key: \"sk-2c3f1f58031b4b86afdb6a8192ea02e2\" # API key (defaults to OPENAI_API_KEY env variable)\n\n # Generation parameters\n temperature: 0.7 # Temperature for generation (higher = more creative)\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n\n # Request parameters\n timeout: 300 # Timeout for API requests in seconds\n retries: 3 # Number of retries for failed requests\n retry_delay: 5 # Delay between retries in seconds\n\n# Prompt configuration\nprompt:\n template_dir: null # Custom directory for prompt templates\n # system_message: \"You are an expert coder helping to improve programs through evolution.\"\n # evaluator_system_message: \"You are an expert code reviewer.\"\n\n # Number of examples to include in the prompt\n num_top_programs: 3 # Number of top-performing programs to include\n num_diverse_programs: 2 # Number of diverse programs to include\n\n # Template stochasticity\n use_template_stochasticity: true # Use random variations in templates for diversity\n template_variations: # Different phrasings for parts of the template\n improvement_suggestion:\n - \"Here's how we could improve this code:\"\n - \"I suggest the following improvements:\"\n - \"We can enhance this code by:\"\n\n # Note: meta-prompting features are not yet implemented\n\n# Database configuration\ndatabase:\n # General settings\n db_path: null # Path to persist database (null = in-memory only)\n in_memory: true # Keep database in memory for faster access\n log_prompts: true # If true, log all prompts and responses into the database\n\n # Evolutionary parameters\n population_size: 1000 # Maximum number of programs to keep in memory\n archive_size: 100 # Size of elite archive\n num_islands: 5 # Number of islands for island model (separate populations)\n\n # Island-based evolution parameters\n # Islands provide diversity by maintaining separate populations that evolve independently.\n # Migration periodically shares the best solutions between adjacent islands.\n migration_interval: 50 # Migrate between islands every N generations\n migration_rate: 0.1 # Fraction of top programs to migrate (0.1 = 10%)\n\n # Selection parameters\n elite_selection_ratio: 0.1 # Ratio of elite programs to select\n exploration_ratio: 0.2 # Ratio of exploration vs exploitation\n exploitation_ratio: 0.7 # Ratio of exploitation vs random selection\n # Note: diversity_metric is fixed to \"edit_distance\" (feature_based not implemented)\n\n # Feature map dimensions for MAP-Elites\n feature_dimensions: # Dimensions for MAP-Elites feature map\n - \"score\" # Performance score\n - \"complexity\" # Code complexity (length)\n feature_bins: 10 # Number of bins per dimension\n\n# Evaluator configuration\nevaluator:\n # General settings\n timeout: 300 # Maximum evaluation time in seconds\n max_retries: 3 # Maximum number of retries for evaluation\n\n # Note: resource limits (memory_limit_mb, cpu_limit) are not yet implemented\n\n # Evaluation strategies\n cascade_evaluation: true # Use cascade evaluation to filter bad solutions early\n cascade_thresholds: # Thresholds for advancing to next evaluation stage\n - 0.5 # First stage threshold\n - 0.75 # Second stage threshold\n - 0.9 # Third stage threshold\n\n # Parallel evaluation\n parallel_evaluations: 4 # Number of parallel evaluations\n # Note: distributed evaluation is not yet implemented\n\n # LLM-based feedback (experimental)\n use_llm_feedback: false # Use LLM to evaluate code quality\n llm_feedback_weight: 0.1 # Weight for LLM feedback in final score\n\n# Reward model configuration\nrewardmodel:\n model_type: api # Model type (vllm or api)\n model_name: deepseek-chat # Model name (if null, uses default)\n temperature: 0.7 # Temperature for generation\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n proposal_score_threshold: 2.0 # Only generate programs if proposal score >= threshold\n api_key: sk-2c3f1f58031b4b86afdb6a8192ea02e2 # API key for API models\n base_url: https://api.deepseek.com # Base URL for API models\n jsonl_file: \"results/reward_results.jsonl\" # JSONL file for results\n max_retries: 50 # Maximum number of retries\n retry_delay: 5 # Delay between retries in seconds\n"}, "ddca181ec5d38f66764a3e312aecf9f36a444cca": {"t": 1785611314.1289542, "n": 6525, "h": "886bb12dc6a6f0de", "c": "# EvolveAgent Default Configuration\n# This file contains all available configuration options with sensible defaults\n# You can use this as a template for your own configuration\n\n# General settings\nmax_iterations: 1000 # Maximum number of evolution iterations\ncheckpoint_interval: 50 # Save checkpoints every N iterations\nlog_level: \"INFO\" # Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)\nlog_dir: null # Custom directory for logs (default: output_dir/logs)\nrandom_seed: null # Random seed for reproducibility (null = random)\n\n# Evolution settings\ndiff_based_evolution: true # Use diff-based evolution (true) or full rewrites (false)\nallow_full_rewrites: false # Allow occasional full rewrites even in diff-based mode\nmax_code_length: 10000 # Maximum allowed code length in characters\n\n# LLM configuration\nllm:\n # Models for evolution\n models:\n # List of available models with their weights\n - name: \"deepseek-chat\"\n weight: 1.0\n # - name: \"gemini-2.0-flash\"\n # weight: 0\n\n # Models for LLM feedback\n # evaluator_models:\n # # List of available models with their weights\n # - name: \"gemini-2.0-flash-lite\"\n # weight: 0.8\n # - name: \"gemini-2.0-flash\"\n # weight: 0.2\n\n # API configuration\n api_base: \"https://api.deepseek.com\" # Base URL for API (change for non-OpenAI models)\n api_key: \"sk-2c3f1f58031b4b86afdb6a8192ea02e2\" # API key (defaults to OPENAI_API_KEY env variable)\n\n # Generation parameters\n temperature: 0.7 # Temperature for generation (higher = more creative)\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n\n # Request parameters\n timeout: 300 # Timeout for API requests in seconds\n retries: 3 # Number of retries for failed requests\n retry_delay: 5 # Delay between retries in seconds\n\n# Prompt configuration\nprompt:\n template_dir: null # Custom directory for prompt templates\n # system_message: \"You are an expert coder helping to improve programs through evolution.\"\n # evaluator_system_message: \"You are an expert code reviewer.\"\n\n # Number of examples to include in the prompt\n num_top_programs: 3 # Number of top-performing programs to include\n num_diverse_programs: 2 # Number of diverse programs to include\n\n # Template stochasticity\n use_template_stochasticity: true # Use random variations in templates for diversity\n template_variations: # Different phrasings for parts of the template\n improvement_suggestion:\n - \"Here's how we could improve this code:\"\n - \"I suggest the following improvements:\"\n - \"We can enhance this code by:\"\n\n # Note: meta-prompting features are not yet implemented\n\n# Database configuration\ndatabase:\n # General settings\n db_path: null # Path to persist database (null = in-memory only)\n in_memory: true # Keep database in memory for faster access\n log_prompts: true # If true, log all prompts and responses into the database\n\n # Evolutionary parameters\n population_size: 1000 # Maximum number of programs to keep in memory\n archive_size: 100 # Size of elite archive\n num_islands: 5 # Number of islands for island model (separate populations)\n\n # Island-based evolution parameters\n # Islands provide diversity by maintaining separate populations that evolve independently.\n # Migration periodically shares the best solutions between adjacent islands.\n migration_interval: 50 # Migrate between islands every N generations\n migration_rate: 0.1 # Fraction of top programs to migrate (0.1 = 10%)\n\n # Selection parameters\n elite_selection_ratio: 0.1 # Ratio of elite programs to select\n exploration_ratio: 0.2 # Ratio of exploration vs exploitation\n exploitation_ratio: 0.7 # Ratio of exploitation vs random selection\n # Note: diversity_metric is fixed to \"edit_distance\" (feature_based not implemented)\n\n # Feature map dimensions for MAP-Elites\n feature_dimensions: # Dimensions for MAP-Elites feature map\n - \"score\" # Performance score\n - \"complexity\" # Code complexity (length)\n feature_bins: 10 # Number of bins per dimension\n\n# Evaluator configuration\nevaluator:\n # General settings\n timeout: 300 # Maximum evaluation time in seconds\n max_retries: 3 # Maximum number of retries for evaluation\n\n # Note: resource limits (memory_limit_mb, cpu_limit) are not yet implemented\n\n # Evaluation strategies\n cascade_evaluation: true # Use cascade evaluation to filter bad solutions early\n cascade_thresholds: # Thresholds for advancing to next evaluation stage\n - 0.5 # First stage threshold\n - 0.75 # Second stage threshold\n - 0.9 # Third stage threshold\n\n # Parallel evaluation\n parallel_evaluations: 4 # Number of parallel evaluations\n # Note: distributed evaluation is not yet implemented\n\n # LLM-based feedback (experimental)\n use_llm_feedback: false # Use LLM to evaluate code quality\n llm_feedback_weight: 0.1 # Weight for LLM feedback in final score\n\n# Reward model configuration\nrewardmodel:\n model_type: api # Model type (vllm or api)\n model_name: deepseek-chat # Model name (if null, uses default)\n temperature: 0.7 # Temperature for generation\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n proposal_score_threshold: 2.0 # Only generate programs if proposal score >= threshold\n api_key: sk-2c3f1f58031b4b86afdb6a8192ea02e2 # API key for API models\n base_url: https://api.deepseek.com # Base URL for API models\n jsonl_file: \"results.jsonl\" # JSONL file for results\n max_retries: 50 # Maximum number of retries\n retry_delay: 5 # Delay between retries in seconds\n"}, "35044ab1cf896aba7216ae2f4e537deb6b84d651": {"t": 1785611316.1975079, "n": 5652, "h": "89836ff792a92866", "c": "# EvolveAgent Default Configuration\n# This file contains all available configuration options with sensible defaults\n# You can use this as a template for your own configuration\n\n# General settings\nmax_iterations: 1000 # Maximum number of evolution iterations\ncheckpoint_interval: 50 # Save checkpoints every N iterations\nlog_level: \"INFO\" # Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)\nlog_dir: null # Custom directory for logs (default: output_dir/logs)\nrandom_seed: null # Random seed for reproducibility (null = random)\n\n# Evolution settings\ndiff_based_evolution: true # Use diff-based evolution (true) or full rewrites (false)\nallow_full_rewrites: false # Allow occasional full rewrites even in diff-based mode\nmax_code_length: 10000 # Maximum allowed code length in characters\n\n# LLM configuration\nllm:\n # Models for evolution\n models:\n # List of available models with their weights\n - name: \"gemini-2.0-flash-lite\"\n weight: 0.8\n - name: \"gemini-2.0-flash\"\n weight: 0.2\n\n # Models for LLM feedback\n evaluator_models:\n # List of available models with their weights\n - name: \"gemini-2.0-flash-lite\"\n weight: 0.8\n - name: \"gemini-2.0-flash\"\n weight: 0.2\n\n # API configuration\n api_base: \"https://generativelanguage.googleapis.com/v1beta/openai/\" # Base URL for API (change for non-OpenAI models)\n api_key: null # API key (defaults to OPENAI_API_KEY env variable)\n\n # Generation parameters\n temperature: 0.7 # Temperature for generation (higher = more creative)\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n\n # Request parameters\n timeout: 60 # Timeout for API requests in seconds\n retries: 3 # Number of retries for failed requests\n retry_delay: 5 # Delay between retries in seconds\n\n# Prompt configuration\nprompt:\n template_dir: null # Custom directory for prompt templates\n system_message: \"You are an expert coder helping to improve programs through evolution.\"\n evaluator_system_message: \"You are an expert code reviewer.\"\n\n # Number of examples to include in the prompt\n num_top_programs: 3 # Number of top-performing programs to include\n num_diverse_programs: 2 # Number of diverse programs to include\n\n # Template stochasticity\n use_template_stochasticity: true # Use random variations in templates for diversity\n template_variations: # Different phrasings for parts of the template\n improvement_suggestion:\n - \"Here's how we could improve this code:\"\n - \"I suggest the following improvements:\"\n - \"We can enhance this code by:\"\n\n # Note: meta-prompting features are not yet implemented\n\n# Database configuration\ndatabase:\n # General settings\n db_path: null # Path to persist database (null = in-memory only)\n in_memory: true # Keep database in memory for faster access\n log_prompts: true # If true, log all prompts and responses into the database\n\n # Evolutionary parameters\n population_size: 1000 # Maximum number of programs to keep in memory\n archive_size: 100 # Size of elite archive\n num_islands: 5 # Number of islands for island model (separate populations)\n\n # Island-based evolution parameters\n # Islands provide diversity by maintaining separate populations that evolve independently.\n # Migration periodically shares the best solutions between adjacent islands.\n migration_interval: 50 # Migrate between islands every N generations\n migration_rate: 0.1 # Fraction of top programs to migrate (0.1 = 10%)\n\n # Selection parameters\n elite_selection_ratio: 0.1 # Ratio of elite programs to select\n exploration_ratio: 0.2 # Ratio of exploration vs exploitation\n exploitation_ratio: 0.7 # Ratio of exploitation vs random selection\n # Note: diversity_metric is fixed to \"edit_distance\" (feature_based not implemented)\n\n # Feature map dimensions for MAP-Elites\n feature_dimensions: # Dimensions for MAP-Elites feature map\n - \"score\" # Performance score\n - \"complexity\" # Code complexity (length)\n feature_bins: 10 # Number of bins per dimension\n\n# Evaluator configuration\nevaluator:\n # General settings\n timeout: 300 # Maximum evaluation time in seconds\n max_retries: 3 # Maximum number of retries for evaluation\n\n # Note: resource limits (memory_limit_mb, cpu_limit) are not yet implemented\n\n # Evaluation strategies\n cascade_evaluation: true # Use cascade evaluation to filter bad solutions early\n cascade_thresholds: # Thresholds for advancing to next evaluation stage\n - 0.5 # First stage threshold\n - 0.75 # Second stage threshold\n - 0.9 # Third stage threshold\n\n # Parallel evaluation\n parallel_evaluations: 4 # Number of parallel evaluations\n # Note: distributed evaluation is not yet implemented\n\n # LLM-based feedback (experimental)\n use_llm_feedback: false # Use LLM to evaluate code quality\n llm_feedback_weight: 0.1 # Weight for LLM feedback in final score\n"}, "e6d64542ff2b87f75108b229d1a68da024a56f5a": {"t": 1785746815.6991138, "n": 6612, "h": "499f610e695c5257", "c": "# EvolveAgent Default Configuration\n# This file contains all available configuration options with sensible defaults\n# You can use this as a template for your own configuration\n\n# General settings\nmax_iterations: 1000 # Maximum number of evolution iterations\ncheckpoint_interval: 50 # Save checkpoints every N iterations\nlog_level: \"INFO\" # Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)\nlog_dir: null # Custom directory for logs (default: output_dir/logs)\nrandom_seed: null # Random seed for reproducibility (null = random)\n\n# Evolution settings\ndiff_based_evolution: true # Use diff-based evolution (true) or full rewrites (false)\nallow_full_rewrites: false # Allow occasional full rewrites even in diff-based mode\nmax_code_length: 100000 # Maximum allowed code length in characters\n\n# LLM configuration\nllm:\n # Models for evolution\n models:\n # List of available models with their weights\n - name: \"deepseek-chat\"\n weight: 1.0\n # - name: \"gemini-2.0-flash\"\n # weight: 0\n\n # Models for LLM feedback\n # evaluator_models:\n # # List of available models with their weights\n # - name: \"gemini-2.0-flash-lite\"\n # weight: 0.8\n # - name: \"gemini-2.0-flash\"\n # weight: 0.2\n\n # API configuration\n api_base: \"https://api.deepseek.com\" # Base URL for API (change for non-OpenAI models)\n api_key: \"sk-2c3f1f58031b4b86afdb6a8192ea02e2\" # API key (defaults to OPENAI_API_KEY env variable)\n\n # Generation parameters\n temperature: 0.7 # Temperature for generation (higher = more creative)\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 8192 # Maximum tokens to generate\n\n # Request parameters\n timeout: 300 # Timeout for API requests in seconds\n retries: 3 # Number of retries for failed requests\n retry_delay: 5 # Delay between retries in seconds\n\n# Prompt configuration\nprompt:\n template_dir: null # Custom directory for prompt templates\n # system_message: \"You are an expert coder helping to improve programs through evolution.\"\n # evaluator_system_message: \"You are an expert code reviewer.\"\n\n # Number of examples to include in the prompt\n num_top_programs: 3 # Number of top-performing programs to include\n num_diverse_programs: 2 # Number of diverse programs to include\n\n # Template stochasticity\n use_template_stochasticity: true # Use random variations in templates for diversity\n template_variations: # Different phrasings for parts of the template\n improvement_suggestion:\n - \"Here's how we could improve this code:\"\n - \"I suggest the following improvements:\"\n - \"We can enhance this code by:\"\n\n # Note: meta-prompting features are not yet implemented\n\n# Database configuration\ndatabase:\n # General settings\n db_path: null # Path to persist database (null = in-memory only)\n in_memory: true # Keep database in memory for faster access\n log_prompts: true # If true, log all prompts and responses into the database\n\n # Evolutionary parameters\n population_size: 1000 # Maximum number of programs to keep in memory\n archive_size: 100 # Size of elite archive\n num_islands: 5 # Number of islands for island model (separate populations)\n\n # Island-based evolution parameters\n # Islands provide diversity by maintaining separate populations that evolve independently.\n # Migration periodically shares the best solutions between adjacent islands.\n migration_interval: 50 # Migrate between islands every N generations\n migration_rate: 0.1 # Fraction of top programs to migrate (0.1 = 10%)\n\n # Selection parameters\n elite_selection_ratio: 0.1 # Ratio of elite programs to select\n exploration_ratio: 0.2 # Ratio of exploration vs exploitation\n exploitation_ratio: 0.7 # Ratio of exploitation vs random selection\n # Note: diversity_metric is fixed to \"edit_distance\" (feature_based not implemented)\n\n # Feature map dimensions for MAP-Elites\n feature_dimensions: # Dimensions for MAP-Elites feature map\n - \"score\" # Performance score\n - \"complexity\" # Code complexity (length)\n feature_bins: 10 # Number of bins per dimension\n\n# Evaluator configuration\nevaluator:\n # General settings\n timeout: 300 # Maximum evaluation time in seconds\n max_retries: 3 # Maximum number of retries for evaluation\n\n # Note: resource limits (memory_limit_mb, cpu_limit) are not yet implemented\n\n # Evaluation strategies\n cascade_evaluation: false # Use cascade evaluation to filter bad solutions early\n cascade_thresholds: # Thresholds for advancing to next evaluation stage\n - 0.5 # First stage threshold\n - 0.75 # Second stage threshold\n - 0.9 # Third stage threshold\n\n # Parallel evaluation\n parallel_evaluations: 4 # Number of parallel evaluations\n # Note: distributed evaluation is not yet implemented\n\n # LLM-based feedback (experimental)\n use_llm_feedback: false # Use LLM to evaluate code quality\n llm_feedback_weight: 0.1 # Weight for LLM feedback in final score\n\n# Reward model configuration\nrewardmodel:\n model_type: vllm # Model type (vllm or api)\n model_name: /data/zhuotaodeng/yzj/alpha_research_model/qwen25_grm_iclr_boxed/checkpoint-180 # Model name (if null, uses default)\n temperature: 0.7 # Temperature for generation\n top_p: 0.95 # Top-p sampling parameter\n max_tokens: 4096 # Maximum tokens to generate\n proposal_score_threshold: 5.5 # Only generate programs if proposal score >= threshold\n # api_key: sk-2c3f1f58031b4b86afdb6a8192ea02e2 # API key for API models\n # base_url: https://api.deepseek.com # Base URL for API models\n jsonl_file: \"results/reward_results.jsonl\" # JSONL file for results\n max_retries: 50 # Maximum number of retries\n retry_delay: 5 # Delay between retries in seconds\n"}}, "benchmark/littlewood_polynomials/initial_program.py": {"f6ede9b919ea90f55f0ffce9efc6ac8704230b4c": {"t": 1785599210.8903413, "n": 814, "h": "531d9750850eb808", "c": "import numpy as np\n\ndef rudin_shapiro(n: int):\n \"\"\"First n signs of the Rudin–Shapiro sequence (±1).\"\"\"\n a = np.ones(n, dtype=int)\n for k in range(n):\n x, cnt, prev = k, 0, 0\n while x:\n b = x & 1\n if b & prev: # saw '11'\n cnt ^= 1\n prev = b\n x >>= 1\n a[k] = 1 if cnt == 0 else -1\n return a\n\ndef random_littlewood(n: int, seed=0):\n rng = np.random.default_rng(seed)\n return rng.choice([-1, 1], size=n).astype(int)\n\ndef main():\n n = 512\n c = rudin_shapiro(n)\n print(f\"n={n}, coeffs={len(c)}\")\n return c\n\nif __name__ == \"__main__\":\n coeffs = main()\n\n# Ensure compatibility with evaluators that expect a global variable\ntry:\n coeffs # type: ignore[name-defined]\nexcept NameError:\n coeffs = main()\n"}}, "benchmark/autoconvolution_peak_minimization/evaluator.py": {"c4df5782cabbb72690dc9ee6480f9929e3dd088e": {"t": 1785599210.9010205, "n": 2425, "h": "9d8113fd66c3c0fd", "c": "import numpy as np\nimport subprocess\nimport sys\nimport traceback\nimport os\nimport json\nfrom typing import Dict\n\ndef evaluate_C1_upper_std(step_heights: np.ndarray) -> Dict[str, float]:\n \"\"\"\n Standard-normalized C1 evaluation function.\n - Project to feasible set: h >= 0 and ∫f = 1 (L1 normalization).\n - Objective: mu_inf = max_t (f*f)(t) (smaller is better).\n \"\"\"\n h = np.asarray(step_heights, dtype=float)\n if h.size == 0 or np.any(h < 0):\n return {\"valid\": 0.0, \"mu_inf\": float(\"inf\"), \"ratio\": float(\"inf\")}\n K = int(len(h))\n dx = 1.0 / K\n integral = float(np.sum(h) * dx)\n if integral <= 0:\n return {\"valid\": 0.0, \"mu_inf\": float(\"inf\"), \"ratio\": float(\"inf\")}\n h = h / integral\n F = np.fft.fft(h, 2*K - 1)\n conv = np.fft.ifft(F * F).real\n conv = np.maximum(conv, 0.0)\n mu_inf = float(np.max(conv) * dx)\n return {\"valid\": 1.0, \"mu_inf\": mu_inf, \"ratio\": mu_inf, \"integral\": 1.0, \"K\": float(K)}\n\ndef evaluate(program_path: str):\n \"\"\"\n Evaluate a program that solves the autoconvolution peak minimization problem.\n \n Returns: dict with key 'score' = 1 / mu_inf (larger is better), or {'error': -1.0}\n \"\"\"\n try:\n import importlib.util\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n step_heights = None\n if hasattr(program, 'step_heights'):\n step_heights = program.step_heights\n elif hasattr(program, 'h'):\n step_heights = program.h\n elif hasattr(program, 'main'):\n result = program.main()\n if isinstance(result, np.ndarray):\n step_heights = result\n elif hasattr(program, 'step_heights'):\n step_heights = program.step_heights\n elif hasattr(program, 'h'):\n step_heights = program.h\n if step_heights is None:\n return {\"error\": -1.0}\n result = evaluate_C1_upper_std(step_heights)\n if result[\"valid\"] == 1.0:\n mu = float(result.get(\"mu_inf\", float(\"inf\")))\n if mu > 0 and np.isfinite(mu):\n return {\"score\": 1.0 / mu}\n return {\"error\": -1.0}\n else:\n return {\"error\": -1.0}\n except Exception as e:\n return {\"error\": -1.0}\n"}}, "benchmark/kissing_number/evaluator.py": {"108de7ab0c0ce19306451c7717377bc97d02c985": {"t": 1785599211.0016513, "n": 3557, "h": "530c2b910dd4eef7", "c": "#@title Verification\nimport numpy as np\nimport subprocess\nimport sys\nimport traceback\nimport os\nimport json\n\n\ndef verify_kissing_configuration(sphere_centers: np.ndarray, atol: float = 1e-9):\n \"\"\"\n Verifies if the given points form a valid kissing number configuration.\n\n A valid kissing configuration of N vectors in D dimensions must satisfy:\n 1. All vectors are unit vectors (norm is 1).\n 2. The dot product of any two distinct vectors is at most 0.5.\n\n Args:\n sphere_centers: A numpy array of shape (N, D) where N is the number of spheres\n and D is the dimension.\n atol: Absolute tolerance for floating point comparisons.\n\n Raises:\n AssertionError: If the configuration is not valid.\n \"\"\"\n num_spheres, dimension = sphere_centers.shape\n \n # 1. Check if all vectors are unit vectors.\n norms = np.linalg.norm(sphere_centers, axis=1)\n assert np.allclose(norms, 1.0, atol=atol), f\"Verification failed: Not all vectors are unit vectors. Norms range from {np.min(norms)} to {np.max(norms)}.\"\n \n # 2. Check the dot products.\n # The dot product of two distinct vectors must be <= 0.5.\n dot_products = sphere_centers @ sphere_centers.T\n \n # We only need to check the upper triangle, excluding the diagonal.\n # The diagonal elements should be 1.0 for unit vectors.\n np.fill_diagonal(dot_products, -np.inf) # so we don't pick diagonal elements\n \n max_dot_product = np.max(dot_products)\n \n # The condition is dot_product <= 0.5\n assert max_dot_product <= 0.5 + atol, f\"Verification failed: Maximum dot product between distinct vectors is {max_dot_product}, which is greater than 0.5.\"\n\n\ndef evaluate(program_path: str):\n \"\"\"\n Evaluate a program that solves the kissing number problem.\n Returns dict with key 'score' = number of spheres (larger is better).\n On failure/invalid, returns {'score': -1.0, ...}.\n \"\"\"\n try:\n # Use importlib.util to dynamically load the program module\n import importlib.util\n \n # Load the module from the given path\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n \n # Look for sphere_centers in the loaded module\n sphere_centers = None\n if hasattr(program, 'sphere_centers'):\n sphere_centers = program.sphere_centers\n elif hasattr(program, 'main'):\n # If there's a main function, try calling it to get sphere_centers\n sphere_centers = program.main()\n \n if sphere_centers is None:\n return {\"score\": -1.0, \"no_sphere_centers\": True}\n \n # Verify the kissing configuration\n verify_kissing_configuration(sphere_centers)\n \n # Calculate metrics\n num_spheres = sphere_centers.shape[0]\n dimension = sphere_centers.shape[1]\n \n # Return metrics with 'score'\n return {\n \"score\": float(num_spheres),\n \"num_spheres\": float(num_spheres),\n \"dimension\": float(dimension)\n }\n \n except Exception as e:\n return {\"score\": -1.0, \"evaluation_error\": True, \"stderr\": traceback.format_exc()}\n\n\nif __name__ == \"__main__\":\n # CLI for debugging: evaluate initial_program.py by default, or a provided path\n try:\n default_path = os.path.join(os.path.dirname(__file__), \"initial_program.py\")\n except Exception:\n default_path = \"initial_program.py\"\n\n target = sys.argv[1] if len(sys.argv) > 1 else default_path\n print(json.dumps(evaluate(target), ensure_ascii=False, indent=2))"}}, "configs/island_config_example.yaml": {"00209c0f1a1dabbbc514d55d7e012bda69134088": {"t": 1785599211.0281053, "n": 1380, "h": "92038c811b2a0765", "c": "# EvolveAgent Island-Based Evolution Configuration\n# This configuration demonstrates the proper use of island-based evolution\n\n# General settings\nmax_iterations: 1000\ncheckpoint_interval: 100\nlog_level: \"INFO\"\n\n# LLM configuration\nllm:\n primary_model: \"gemini-2.0-flash-lite\"\n primary_model_weight: 0.8\n secondary_model: \"gemini-2.0-flash\"\n secondary_model_weight: 0.2\n temperature: 0.7\n top_p: 0.95\n max_tokens: 4096\n\n# Database configuration with proper island settings\ndatabase:\n population_size: 500\n archive_size: 100\n \n # Island-based evolution settings\n num_islands: 5 # Number of separate populations\n migration_interval: 50 # Migrate every 50 generations\n migration_rate: 0.1 # Migrate 10% of top programs\n \n # Selection parameters\n elite_selection_ratio: 0.1\n exploration_ratio: 0.3\n exploitation_ratio: 0.7\n # Note: diversity_metric fixed to \"edit_distance\"\n \n # Feature map dimensions for MAP-Elites\n feature_dimensions: [\"score\", \"complexity\"]\n feature_bins: 10\n\n# Prompt configuration\nprompt:\n num_top_programs: 3\n num_diverse_programs: 2\n use_template_stochasticity: true\n\n# Evaluator configuration\nevaluator:\n timeout: 300\n max_retries: 3\n cascade_evaluation: true\n parallel_evaluations: 4\n\n# Evolution settings\ndiff_based_evolution: true\nallow_full_rewrites: false\nmax_code_length: 10000\n"}}, "benchmark/MSTD/evaluator.py": {"679c960f99963d806d6f5b8483387394c8d37e30": {"t": 1785599211.0464013, "n": 2351, "h": "5f9087e22c08f49c", "c": "import numpy as np\nimport importlib.util\nimport sys\nimport os\nimport json\n\n\ndef _to_index_set(indicators):\n arr = np.asarray(indicators).astype(int)\n return np.nonzero(arr)[0]\n\n\ndef mstd_ratio(A_idx, B_idx=None):\n \"\"\"\n Classical MSTD ratio with enforced A=B: R = |A+A| / |A−A|.\n Any provided B is ignored to keep consistency with the baseline setting.\n \"\"\"\n # Enforce classic setting: ignore B and set B_idx = A_idx\n B_idx = A_idx\n if len(A_idx) == 0 or len(B_idx) == 0:\n return -1.0\n sumset, diffset = set(), set()\n for a in A_idx:\n for b in B_idx:\n sumset.add(int(a + b))\n diffset.add(int(a - b))\n if len(diffset) == 0:\n return -1.0\n return float(len(sumset)) / float(len(diffset))\n\n\ndef evaluate(program_path: str):\n try:\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n\n # Accept either A_indicators (and optional B_indicators) or a main() returning them\n A = None\n B = None\n if hasattr(program, 'A_indicators'):\n A = program.A_indicators\n if hasattr(program, 'B_indicators'):\n B = program.B_indicators\n if A is None:\n if hasattr(program, 'main'):\n res = program.main()\n if isinstance(res, tuple) and len(res) in (1, 2):\n if len(res) == 1:\n A = res[0]\n B = None\n else:\n A, B = res\n if A is None:\n return {\"error\": -1.0}\n\n A_idx = _to_index_set(A)\n # Enforce classic setting regardless of provided B\n R = mstd_ratio(A_idx, None)\n if R <= 0:\n return {\"error\": -1.0}\n # Higher is better: score = R\n return {\"score\": float(R), \"ratio\": float(R)}\n except Exception:\n return {\"error\": -1.0}\n\n\nif __name__ == \"__main__\":\n try:\n default_path = os.path.join(os.path.dirname(__file__), \"initial_program.py\")\n except Exception:\n default_path = \"initial_program.py\"\n target = sys.argv[1] if len(sys.argv) > 1 else default_path\n print(json.dumps(evaluate(target), ensure_ascii=False))\n\n\n"}}, "benchmark/heilbronn_in_the_unit_square/visualization.py": {"56ab246e3252ac0bd7fb94ae4c7fe820bbdb964f": {"t": 1785599211.0584717, "n": 6366, "h": "4402837f17c312d0", "c": "# viz_min_triangle.py\nimport argparse\nimport importlib.util\nimport json\nimport os\nimport sys\nfrom itertools import combinations\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n\n# ---------- 评测与辅助函数（与您 evaluator 一致/兼容） ----------\n\ndef _triangle_area(a, b, c) -> float:\n return abs((b[0]-a[0])*(c[1]-a[1]) - (b[1]-a[1])*(c[0]-a[0])) * 0.5\n\ndef find_min_triangle(points: np.ndarray):\n \"\"\"\n 返回最小三角形：(i, j, k, min_area)\n 若点数<3，返回 (-1, -1, -1, 0.0)\n \"\"\"\n P = np.asarray(points, dtype=float)\n n = len(P)\n if n < 3:\n return -1, -1, -1, 0.0\n best = (-1, -1, -1, float(\"inf\"))\n for i, j, k in combinations(range(n), 3):\n area = _triangle_area(P[i], P[j], P[k])\n if area < best[3]:\n best = (i, j, k, area)\n if area == 0.0:\n break\n return best\n\ndef evaluate_min_triangle_area(points: np.ndarray):\n \"\"\"\n 与您当前 evaluator 保持一致的指标：\n - min_area：最小三角形面积（越大越好）\n - scaled_min_area：n^(8/7 + 1/2000) * min_area\n - score：等于 min_area（越大越好）\n \"\"\"\n pts = np.asarray(points, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 2 or len(pts) < 3:\n return dict(valid=0.0, min_area=0.0, n=float(len(pts)),\n scaled_min_area=0.0, score=0.0, argmin_triplet=(-1,-1,-1))\n i, j, k, min_area = find_min_triangle(pts)\n n = float(len(pts))\n exponent = (8.0/7.0) + (1.0/2000.0)\n scaled_min_area = (n ** exponent) * float(min_area)\n return dict(\n valid=1.0,\n min_area=float(min_area),\n n=n,\n scaled_min_area=float(scaled_min_area),\n score=float(min_area),\n argmin_triplet=(int(i), int(j), int(k))\n )\n\n\n# ---------- 读取点数据（模块 / npy / csv） ----------\n\ndef load_from_module(module_path: str) -> np.ndarray:\n module_path = os.path.abspath(module_path)\n spec = importlib.util.spec_from_file_location(\"points_mod\", module_path)\n if spec is None or spec.loader is None:\n raise RuntimeError(f\"无法加载模块: {module_path}\")\n mod = importlib.util.module_from_spec(spec)\n spec.loader.exec_module(mod)\n\n pts = None\n if hasattr(mod, \"points\"):\n pts = mod.points\n elif hasattr(mod, \"main\"):\n res = mod.main()\n try:\n pts = np.asarray(res, dtype=float)\n except Exception:\n pass\n if pts is None and hasattr(mod, \"points\"):\n pts = mod.points\n if pts is None:\n raise RuntimeError(\"模块中既无 `points` 变量，也无法从 `main()` 获取点。\")\n\n pts = np.asarray(pts, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 2:\n raise ValueError(f\"模块返回的点形状异常: {pts.shape}, 期望 (N,2)\")\n return pts\n\ndef load_from_npy(npy_path: str) -> np.ndarray:\n pts = np.load(npy_path)\n pts = np.asarray(pts, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 2:\n raise ValueError(f\"npy 形状异常: {pts.shape}, 期望 (N,2)\")\n return pts\n\ndef load_from_csv(csv_path: str) -> np.ndarray:\n pts = np.loadtxt(csv_path, delimiter=\",\")\n pts = np.asarray(pts, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 2:\n raise ValueError(f\"csv 形状异常: {pts.shape}, 期望 (N,2)\")\n return pts\n\n\n# ---------- 可视化 ----------\n\ndef plot_points_and_min_triangle(points: np.ndarray,\n show_indices: bool = False,\n title_prefix: str = \"\"):\n pts = np.asarray(points, dtype=float)\n (i, j, k, amin) = find_min_triangle(pts)\n eval_res = evaluate_min_triangle_area(pts)\n\n fig, ax = plt.subplots(figsize=(6, 6))\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n ax.set_aspect(\"equal\", adjustable=\"box\")\n\n # 画所有点\n ax.scatter(pts[:, 0], pts[:, 1], s=40, zorder=2)\n\n # 可选：标注索引\n if show_indices:\n for idx, (x, y) in enumerate(pts):\n ax.text(x, y, str(idx), fontsize=9, ha=\"left\", va=\"bottom\")\n\n # 高亮最小三角形\n if i >= 0:\n tri = np.array([pts[i], pts[j], pts[k], pts[i]])\n ax.plot(tri[:, 0], tri[:, 1], linewidth=2.5, zorder=3)\n ax.scatter(pts[[i, j, k], 0], pts[[i, j, k], 1], s=70, zorder=4)\n\n # 网格与边框\n ax.set_xticks(np.linspace(0, 1, 6))\n ax.set_yticks(np.linspace(0, 1, 6))\n ax.grid(True, linestyle=\"--\", alpha=0.3)\n\n # 标题（包含指标）\n title = (\n f\"{title_prefix}min_area={eval_res['min_area']:.8f} | \"\n f\"scaled_min_area={eval_res['scaled_min_area']:.6f} | \"\n f\"score={eval_res['score']:.8f} | \"\n f\"argmin={eval_res['argmin_triplet']}\"\n )\n ax.set_title(title)\n plt.tight_layout()\n plt.show()\n\n # 同时在 stdout 打一份 JSON，方便脚本化调用时抓数值\n out = {\n \"min_area\": eval_res[\"min_area\"],\n \"scaled_min_area\": eval_res[\"scaled_min_area\"],\n \"score\": eval_res[\"score\"],\n \"argmin_triplet\": eval_res[\"argmin_triplet\"],\n \"n\": int(eval_res[\"n\"]),\n }\n print(json.dumps(out, ensure_ascii=False, indent=2))\n\n\n# ---------- CLI ----------\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"可视化 [0,1]^2 中点集，并高亮最小三角形（支持模块 / .npy / .csv）。若未提供来源，将自动尝试读取同目录下的 points.npy\"\n )\n src = parser.add_mutually_exclusive_group(required=False)\n src.add_argument(\"--from-module\", type=str, help=\"含 points 或 main() 的 Python 文件路径\")\n src.add_argument(\"--from-npy\", type=str, help=\"N×2 的 .npy 路径\")\n src.add_argument(\"--from-csv\", type=str, help=\"N×2 的 .csv 路径（逗号分隔）\")\n parser.add_argument(\"--show-indices\", action=\"store_true\", help=\"是否标注点索引\")\n parser.add_argument(\"--title\", type=str, default=\"\", help=\"标题前缀\")\n args = parser.parse_args()\n\n if args.from_module:\n P = load_from_module(args.from_module)\n elif args.from_npy:\n P = load_from_npy(args.from_npy)\n elif args.from_csv:\n P = load_from_csv(args.from_csv)\n else:\n # 自动读取默认的 points.npy（位于本脚本同目录）\n default_path = os.path.join(os.path.dirname(__file__), \"points.npy\")\n if not os.path.exists(default_path):\n print(\n \"未提供输入来源，且未在本目录找到 points.npy。请先运行 initial_program.py 生成 points.npy，或通过 --from-* 指定输入。\",\n file=sys.stderr,\n )\n sys.exit(2)\n P = load_from_npy(default_path)\n\n # 可选：如果任务要求必须在 [0,1]^2，可以做个提示（不改变数值）\n if not (np.all(P >= 0.0) and np.all(P <= 1.0)):\n print(\"⚠️ 警告：存在越界点（不在 [0,1]^2），图中仍会显示。\", file=sys.stderr)\n\n plot_points_and_min_triangle(P, show_indices=args.show_indices, title_prefix=args.title)\n\nif __name__ == \"__main__\":\n main()\n"}}, "benchmark/littlewood_polynomials/evaluator.py": {"c61904eb3554f9303e4a60cef31de674ed38ae7e": {"t": 1785599211.0596972, "n": 2824, "h": "c76431a63e6ce4d3", "c": "import sys\nimport re\nimport os\nimport json\nimport importlib.util\nimport numpy as np\nfrom typing import Dict\n\ndef evaluate_littlewood_supnorm(coeffs, num_grid: int = 16384) -> Dict[str, float]:\n \"\"\"\n FFT-sampled sup-norm upper bound on |P(e^{it})|.\n - coeffs: 1-D array-like of ±1\n - num_grid: sampling resolution (larger -> tighter upper bound)\n \"\"\"\n if num_grid < 8:\n raise ValueError(\"num_grid too small\")\n c = np.atleast_1d(np.asarray(coeffs, dtype=float)) # ensure 1-D\n if c.ndim != 1 or c.size == 0:\n raise ValueError(\"coeffs must be a non-empty 1-D array\")\n\n pad = np.zeros(int(num_grid), dtype=np.complex128)\n pad[: c.size] = c.astype(np.complex128)\n\n values = np.fft.fft(pad) # samples of P on unit circle\n supnorm = float(np.max(np.abs(values)))\n return {\"valid\": 1.0, \"supnorm\": supnorm}\n\ndef _read_coeffs_from_stdin():\n \"\"\"\n Accepts lines like:\n n = 512\n 1 -1 1 1 -1 ...\n or just a line of ±1's. Robust to extra spaces/newlines.\n \"\"\"\n text = sys.stdin.read().strip()\n if not text:\n return None\n # Take the last line that contains numbers\n lines = [ln.strip() for ln in text.splitlines() if ln.strip()]\n # concatenate all non \"n =\" lines to support long sequences split across lines\n number_lines = [ln for ln in lines if not ln.lower().startswith(\"n =\")]\n if not number_lines:\n return None\n joined = \" \".join(number_lines)\n nums = re.findall(r\"[-+]?\\d+\", joined)\n return np.asarray(list(map(int, nums)), dtype=int)\n\ndef evaluate(program_path: str = \"littlewood_polynomials/initial_program.py\"):\n try:\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n\n coeffs_obj = None\n if hasattr(program, 'coeffs'):\n coeffs_obj = program.coeffs\n elif hasattr(program, 'main'):\n res = program.main()\n if isinstance(res, np.ndarray):\n coeffs_obj = res\n elif hasattr(program, 'coeffs'):\n coeffs_obj = program.coeffs\n if coeffs_obj is None:\n # fallback: try stdin for robustness\n coeffs_obj = _read_coeffs_from_stdin()\n if coeffs_obj is None:\n return {\"error\": -1.0}\n result = evaluate_littlewood_supnorm(coeffs_obj, num_grid=16384)\n if result.get(\"valid\", 0.0) != 1.0:\n return {\"error\": -1.0}\n supnorm = float(result[\"supnorm\"])\n if supnorm > 0 and np.isfinite(supnorm):\n return {\"score\": 1.0 / supnorm}\n return {\"error\": -1.0}\n except Exception:\n return {\"error\": -1.0}\n\nprint(evaluate())"}}, "benchmark/autoconvolution_peak_minimization/initial_program.py": {"91318d03476d58c27285e81d0fa79251e3bcf274": {"t": 1785599211.2383194, "n": 2527, "h": "466a2678fb0b8d95", "c": "\n#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nAutoconvolution Peak Minimization\n=================================\n\nThis program generates step heights for a probability density function\nthat minimizes the maximum value of its autoconvolution.\n\"\"\"\n\nimport numpy as np\nfrom typing import Dict\n\ndef evaluate_C1_upper_std(step_heights: np.ndarray) -> Dict[str, float]:\n \"\"\"\n Standard-normalized C1 (support [-1/2,1/2], dx=1/K).\n - Project to feasible set: h >= 0 and ∫f = 1 (L1 normalization).\n - Objective: mu_inf = max_t (f*f)(t) (smaller is better).\n Returns: {\"valid\", \"mu_inf\", \"ratio\"(=mu_inf), \"integral\"(=1.0), \"K\"}\n \"\"\"\n h = np.asarray(step_heights, dtype=float)\n if h.size == 0 or np.any(h < 0):\n return {\"valid\": 0.0, \"mu_inf\": float(\"inf\"), \"ratio\": float(\"inf\")}\n K = int(len(h))\n dx = 1.0 / K\n\n integral = float(np.sum(h) * dx)\n if integral <= 0:\n return {\"valid\": 0.0, \"mu_inf\": float(\"inf\"), \"ratio\": float(\"inf\")}\n h = h / integral # ∫f = 1\n\n F = np.fft.fft(h, 2*K - 1) # linear autoconvolution via padding\n conv = np.fft.ifft(F * F).real\n conv = np.maximum(conv, 0.0) # clamp tiny negatives\n\n mu_inf = float(np.max(conv) * dx)\n return {\"valid\": 1.0, \"mu_inf\": mu_inf, \"ratio\": mu_inf, \"integral\": 1.0, \"K\": float(K)}\n\ndef make_candidate(K: int, kind: str = \"cos2\") -> np.ndarray:\n \"\"\"\n Simple candidate builder on [-1/2,1/2] (NOT normalized here).\n \n Args:\n K: Number of discretization points\n kind: Type of candidate function (\"box\", \"triangle\", \"cos2\", \"gauss\")\n \n Returns:\n Step heights array\n \"\"\"\n x = np.linspace(-1.0, 1.0, K)\n if kind == \"box\":\n h = np.ones(K)\n elif kind == \"triangle\":\n h = 1.0 - np.abs(x)\n h[h < 0] = 0.0\n elif kind == \"cos2\":\n h = np.cos(np.pi * x / 2.0) ** 2\n elif kind == \"gauss\":\n h = np.exp(-4.0 * x**2)\n else:\n raise ValueError(f\"unknown kind={kind}\")\n return h\n\ndef main():\n \"\"\"\n Main function that generates step heights for autoconvolution minimization.\n \n Returns:\n numpy.ndarray: Step heights array\n \"\"\"\n K = 128\n kind = \"cos2\" # Change this to try different candidates (box/triangle/cos2/gauss)\n step_heights = make_candidate(K, kind)\n \n # Evaluate the result to verify it's valid\n result = evaluate_C1_upper_std(step_heights)\n print(f\"Generated {kind} candidate with K={K}, mu_inf={result['mu_inf']:.6f}\")\n \n return step_heights\n"}}, "benchmark/heilbronn_in_the_unit_square/initial_program.py": {"166e26fdcb42433299ec3374e6e252f6e047a9e3": {"t": 1785599211.292773, "n": 10472, "h": "a43996b262396869", "c": "import numpy as np\nimport os\nimport time\nimport argparse\nfrom itertools import combinations\n\n# =========================\n# 基本几何与评估\n# =========================\n\ndef triangle_area(a, b, c) -> float:\n # 三角形面积 = |(b-a) x (c-a)|/2\n return abs((b[0]-a[0])*(c[1]-a[1]) - (b[1]-a[1])*(c[0]-a[0])) * 0.5\n\n_triplets_cache = {}\n\ndef _precompute_triplets(n: int):\n from itertools import combinations as _comb\n combs = np.array(list(_comb(range(n), 3)), dtype=int)\n I, J, K = combs[:, 0], combs[:, 1], combs[:, 2]\n return I, J, K, combs\n\ndef find_min_triangle(P: np.ndarray):\n \"\"\"向量化求最小三角形：返回 (i,j,k,min_area)。若 n<3 返回 (-1,-1,-1,0.0)。\"\"\"\n n = len(P)\n if n < 3:\n return -1, -1, -1, 0.0\n if n not in _triplets_cache:\n _triplets_cache[n] = _precompute_triplets(n)\n I, J, K, combs = _triplets_cache[n]\n A = P[I]\n B = P[J]\n C = P[K]\n area = np.abs((B[:,0]-A[:,0])*(C[:,1]-A[:,1]) - (B[:,1]-A[:,1])*(C[:,0]-A[:,0])) * 0.5\n if area.size == 0:\n return -1, -1, -1, 0.0\n idx = int(np.argmin(area))\n i, j, k = combs[idx]\n return int(i), int(j), int(k), float(area[idx])\n\ndef min_triangle_area(P: np.ndarray) -> float:\n return find_min_triangle(P)[3]\n\ndef scaled_min_area(P: np.ndarray) -> float:\n n = float(len(P))\n exp = (8.0/7.0) + (1.0/2000.0)\n return (n ** exp) * min_triangle_area(P)\n\n# =========================\n# 初始化：多起点\n# =========================\n\ndef jittered_grid_points(n, seed=0):\n rng = np.random.default_rng(seed)\n m = int(round(np.sqrt(n))); m = max(m, 2)\n xs = (np.arange(m) + 0.5) / m\n ys = (np.arange(m) + 0.5) / m\n X, Y = np.meshgrid(xs, ys)\n P = np.c_[X.ravel(), Y.ravel()]\n jitter = 0.12 / m\n P += rng.uniform(-jitter, jitter, size=P.shape)\n P = np.clip(P[:n], 0.0, 1.0)\n return P\n\ndef hex_lattice_points(n, seed=0):\n rng = np.random.default_rng(seed)\n a = 1.0 / np.sqrt(n)\n pts = []\n y = a/2\n row = 0\n while y < 1.0:\n x0 = (a/2) if (row % 2 == 1) else a\n x = x0\n while x < 1.0:\n pts.append([x, y])\n x += a\n y += np.sqrt(3)/2 * a\n row += 1\n P = np.array(pts, dtype=float)\n if len(P) < n:\n extra = rng.uniform(0, 1, size=(n - len(P), 2))\n P = np.vstack([P, extra])\n P = P[:n]\n P += rng.uniform(-0.08*a, 0.08*a, size=P.shape)\n P = np.clip(P, 0.0, 1.0)\n return P\n\ndef bridson_poisson_disk(n, r=None, k=30, seed=0):\n \"\"\"\n 近似生成 >=n 的 Poisson-disk 点，再均匀抽样到 n 个。\n r: 目标最小间距 ~ c / sqrt(n)\n \"\"\"\n rng = np.random.default_rng(seed)\n if r is None:\n r = 0.6 / np.sqrt(n) # 稍保守的间距\n cell_size = r / np.sqrt(2)\n grid_w = int(np.ceil(1.0 / cell_size))\n grid_h = int(np.ceil(1.0 / cell_size))\n grid = -np.ones((grid_h, grid_w), dtype=int)\n\n def grid_coords(pt):\n return int(pt[1] / cell_size), int(pt[0] / cell_size)\n\n def in_neighborhood(pt):\n gy, gx = grid_coords(pt)\n for yy in range(max(gy-2,0), min(gy+3, grid_h)):\n for xx in range(max(gx-2,0), min(gx+3, grid_w)):\n j = grid[yy, xx]\n if j >= 0:\n if np.linalg.norm(pts[j] - pt) < r:\n return True\n return False\n\n pts = []\n active = []\n\n # 初始点\n p0 = rng.uniform(0, 1, size=2)\n pts.append(p0); active.append(0)\n gy, gx = grid_coords(p0); grid[gy, gx] = 0\n\n while active and len(pts) < max(n*2, n+10):\n idx = rng.choice(active)\n base = pts[idx]\n found = False\n for _ in range(k):\n rad = rng.uniform(r, 2*r)\n ang = rng.uniform(0, 2*np.pi)\n cand = base + rad * np.array([np.cos(ang), np.sin(ang)])\n if not (0 <= cand[0] <= 1 and 0 <= cand[1] <= 1):\n continue\n if not in_neighborhood(cand):\n pts.append(cand)\n gy, gx = grid_coords(cand); grid[gy, gx] = len(pts)-1\n active.append(len(pts)-1)\n found = True\n break\n if not found:\n active.remove(idx)\n\n pts = np.array(pts)\n if len(pts) >= n:\n idx = rng.choice(len(pts), size=n, replace=False)\n pts = pts[idx]\n else:\n extra = rng.uniform(0,1,size=(n-len(pts),2))\n pts = np.vstack([pts, extra])\n return pts\n\n# =========================\n# 定向局部搜索（增大最小三角形）\n# =========================\n\ndef normalize(v):\n n = np.linalg.norm(v)\n return v / n if n > 1e-12 else v\n\ndef bump_point(P, idx, step, rng):\n \"\"\"对单个点做小扰动（含随机 + 轻微向内推），保持在 [0,1]^2。\"\"\"\n q = P.copy()\n jitter = rng.uniform(-step, step, size=2)\n # 轻微向内推，减少贴边导致的瘦三角形\n inward = 0.15 * step * (0.5 - P[idx])\n q[idx] = np.clip(P[idx] + jitter + inward, 0.0, 1.0)\n return q\n\ndef bump_min_triangle_directed(P, step, rng):\n \"\"\"\n 针对“当前最小三角形”的三个顶点，沿增大面积的几何方向优先移动：\n - 对顶点 a，相对边 (b,c) 的法向方向能增大面积。\n - 叠加小随机扰动，避免卡鞍点。\n \"\"\"\n i, j, k, _ = find_min_triangle(P)\n if i < 0:\n return P\n a, b, c = P[i], P[j], P[k]\n\n def move_along_normal(P, idx, other1, other2):\n q = P.copy()\n base = other2 - other1\n # 2D 中与 base 垂直的法向（取两种方向试探）\n n1 = normalize(np.array([ base[1], -base[0] ]))\n n2 = -n1\n cand1 = np.clip(P[idx] + step*n1, 0.0, 1.0)\n cand2 = np.clip(P[idx] + step*n2, 0.0, 1.0)\n # 选择带来更大 min_area 的方向\n q1 = q.copy(); q1[idx] = cand1\n q2 = q.copy(); q2[idx] = cand2\n a1 = min_triangle_area(q1); a2 = min_triangle_area(q2)\n if a1 >= a2:\n return q1, a1\n else:\n return q2, a2\n\n # 依次尝试移动 i、j、k，并保留最好者\n bestP = P.copy(); bestA = min_triangle_area(P)\n for (idx, o1, o2) in [(i, b, c), (j, c, a), (k, a, b)]:\n q, area_dir = move_along_normal(bestP, idx, o1, o2)\n if area_dir > bestA + 1e-15:\n bestP, bestA = q, area_dir\n else:\n # 若定向无改进，退而求其次：随机小扰动\n q = bump_point(bestP, idx, 0.6*step, rng)\n a_rand = min_triangle_area(q)\n if a_rand > bestA + 1e-15:\n bestP, bestA = q, a_rand\n return bestP\n\ndef project_min_distance(P, dmin=1e-3, iters=1):\n \"\"\"软约束：尽量避免过近点对（简单排斥迭代）。\"\"\"\n Q = P.copy()\n for _ in range(iters):\n for i in range(len(Q)):\n diffs = Q - Q[i]\n dists = np.linalg.norm(diffs, axis=1)\n mask = (dists < dmin) & (dists > 0)\n if np.any(mask):\n repel = -diffs[mask]\n move = 0.5 * np.sum(repel / np.maximum(dists[mask][:,None], 1e-12), axis=0)\n Q[i] = np.clip(Q[i] + 1e-3*move, 0.0, 1.0)\n return Q\n\ndef improve(P0, iters=6000, step0=0.05, seed=0, patience=800, time_limit=None):\n \"\"\"\n 退火式定向搜索：\n - 以“当前最小三角形”为线索，优先移动那三个点；\n - 步长逐步衰减并穿插随机扰动与最小距离投影；\n \"\"\"\n rng = np.random.default_rng(seed)\n P = np.clip(P0.copy(), 0.0, 1.0)\n bestP = P.copy(); bestA = min_triangle_area(P)\n no_improve = 0\n t0 = time.time()\n\n step = step0\n for t in range(1, iters+1):\n if time_limit is not None and (time.time() - t0) > time_limit:\n break\n Q = bump_min_triangle_directed(P, step, rng)\n # 偶尔对非最小三角形顶点做随机扰动，避免局部陷阱\n if t % 30 == 0:\n idx = rng.integers(len(P))\n Q = bump_point(Q, idx, 0.5*step, rng)\n\n # 软性分离，避免过近\n if t % 50 == 0:\n Q = project_min_distance(Q, dmin=5e-3, iters=1)\n\n aQ = min_triangle_area(Q)\n if aQ > bestA + 1e-15:\n P = Q\n bestP, bestA = Q.copy(), aQ\n no_improve = 0\n else:\n # 以小概率接受较差解可加入，但这里保守：不接受\n no_improve += 1\n\n if no_improve >= patience:\n break\n\n # 步长衰减\n if t % 400 == 0:\n step *= 0.7\n step = max(step, 5e-4)\n\n return bestP, bestA\n\n# =========================\n# 主流程：多起点 + 精修\n# =========================\n\ndef multi_start_optimize(n=16, seeds=(42, 43, 44), iters=6000, step0=0.05, time_limit=None):\n \"\"\"\n 多路起点（Hex / Grid / Poisson）并行，保留最好的，再额外精修一轮。\n \"\"\"\n cands = []\n for s in seeds:\n cands.append(hex_lattice_points(n, seed=s))\n cands.append(jittered_grid_points(n, seed=1000+s))\n cands.append(bridson_poisson_disk(n, seed=2000+s))\n bestP = None; bestA = -1.0\n t0 = time.time()\n # 粗搜索：较少迭代，快速筛选\n coarse_iters = max(200, int(0.25 * iters))\n coarse_results = []\n for P0 in cands:\n remaining = None\n if time_limit is not None:\n elapsed = time.time() - t0\n remaining = max(0.0, time_limit - elapsed)\n if remaining <= 0:\n break\n P1, A1 = improve(P0, iters=coarse_iters, step0=step0, seed=12345, time_limit=remaining)\n coarse_results.append((A1, P1))\n\n if coarse_results:\n coarse_results.sort(key=lambda x: x[0], reverse=True)\n top_list = [P for (_, P) in coarse_results[:3]]\n else:\n top_list = cands[:1]\n\n # 精修：更小步长\n for idx, P0 in enumerate(top_list):\n remaining = None\n if time_limit is not None:\n elapsed = time.time() - t0\n remaining = max(0.0, time_limit - elapsed)\n if remaining <= 0:\n break\n P2, A2 = improve(P0, iters=max(400, int(0.6 * iters)), step0=0.02, seed=999+idx, time_limit=remaining)\n if A2 > bestA:\n bestP, bestA = P2, A2\n return bestP, bestA\n\n# =========================\n# 入口：生成 points 并保存\n# =========================\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--iters\", type=int, default=2000)\n parser.add_argument(\"--step0\", type=float, default=0.04)\n parser.add_argument(\"--seeds\", type=str, default=\"7,11,19\")\n parser.add_argument(\"--time-limit\", type=float, default=None)\n args = parser.parse_args()\n\n n = 16\n seeds = tuple(int(s.strip()) for s in args.seeds.split(\",\") if s.strip()) or (7, 11, 19)\n bestP, bestA = multi_start_optimize(n=n, seeds=seeds, iters=args.iters, step0=args.step0, time_limit=args.time_limit)\n smin = scaled_min_area(bestP)\n print(f\"n={n}, points={len(bestP)}\")\n print(f\"min_area = {bestA:.10f}\")\n print(f\"scaled_min_area = {smin:.10f}\")\n return bestP\n\nif __name__ == \"__main__\":\n points = main()\n out_path = os.path.join(os.path.dirname(__file__), \"points.npy\")\n np.save(out_path, points)\n print(f\"Saved points to {out_path}\")\n\n# 兼容外部 evaluator\ntry:\n points # type: ignore[name-defined]\nexcept NameError:\n points = main()\n"}}, "evolve_agent/config.py": {"2c4a0f80ed3a24db83ab67438979708ccb2685bb": {"t": 1785599211.390441, "n": 14778, "h": "9402899e369b57b8", "c": "\"\"\"\nConfiguration handling for EvolveAgent\n\"\"\"\n\nimport os\nfrom dataclasses import dataclass, field\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Union\n\nimport yaml\n\n\n@dataclass\nclass LLMModelConfig:\n \"\"\"Configuration for a single LLM model\"\"\"\n\n # API configuration\n api_base: str = None\n api_key: Optional[str] = None\n name: str = None\n\n # Weight for model in ensemble\n weight: float = 1.0\n\n # Generation parameters\n system_message: Optional[str] = None\n temperature: float = None\n top_p: float = None\n max_tokens: int = None\n\n # Request parameters\n timeout: int = None\n retries: int = None\n retry_delay: int = None\n\n\n@dataclass\nclass LLMConfig(LLMModelConfig):\n \"\"\"Configuration for LLM models\"\"\"\n\n # API configuration\n api_base: str = \"https://api.openai.com/v1\"\n\n # Generation parameters\n system_message: Optional[str] = \"system_message\"\n temperature: float = 0.7\n top_p: float = 0.95\n max_tokens: int = 4096\n\n # Request parameters\n timeout: int = 60\n retries: int = 3\n retry_delay: int = 5\n\n # n-model configuration for evolution LLM ensemble\n models: List[LLMModelConfig] = field(default_factory=lambda: [LLMModelConfig()])\n\n # n-model configuration for evaluator LLM ensemble\n evaluator_models: List[LLMModelConfig] = field(default_factory=lambda: [])\n\n # Backwardes compatibility with primary_model(_weight) options\n primary_model: str = None\n primary_model_weight: float = None\n secondary_model: str = None\n secondary_model_weight: float = None\n\n def __post_init__(self):\n \"\"\"Post-initialization to set up model configurations\"\"\"\n # Handle backward compatibility for primary_model(_weight) and secondary_model(_weight).\n if (self.primary_model or self.primary_model_weight) and len(self.models) < 1:\n # Ensure we have a primary model\n self.models.append(LLMModelConfig())\n if self.primary_model:\n self.models[0].name = self.primary_model\n if self.primary_model_weight:\n self.models[0].weight = self.primary_model_weight\n\n if (self.secondary_model or self.secondary_model_weight) and len(self.models) < 2:\n # Ensure we have a second model\n self.models.append(LLMModelConfig())\n if self.secondary_model:\n self.models[1].name = self.secondary_model\n if self.secondary_model_weight:\n self.models[1].weight = self.secondary_model_weight\n\n # If no evaluator models are defined, use the same models as for evolution\n if not self.evaluator_models or len(self.evaluator_models) < 1:\n self.evaluator_models = self.models.copy()\n\n # Update models with shared configuration values\n shared_config = {\n \"api_base\": self.api_base,\n \"api_key\": self.api_key,\n \"temperature\": self.temperature,\n \"top_p\": self.top_p,\n \"max_tokens\": self.max_tokens,\n \"timeout\": self.timeout,\n \"retries\": self.retries,\n \"retry_delay\": self.retry_delay,\n }\n self.update_model_params(shared_config)\n\n def update_model_params(self, args: Dict[str, Any], overwrite: bool = False) -> None:\n \"\"\"Update model parameters for all models\"\"\"\n for model in self.models + self.evaluator_models:\n for key, value in args.items():\n if overwrite or getattr(model, key, None) is None:\n setattr(model, key, value)\n\n@dataclass\nclass RewardModelConfig:\n \"\"\"Configuration for reward model\"\"\"\n\n model_type: str = \"vllm\"\n model_name: str = None\n\n # Generation parameters\n temperature: float = 0.7\n top_p: float = 0.95\n max_tokens: int = 4096\n \n # Threshold for generating programs\n proposal_score_threshold: float = 5.5 # Only generate programs if proposal score >= threshold\n\n # Request parameters of API models\n api_key: Optional[str] = None,\n base_url: Optional[str] = None,\n jsonl_file: str = \"results.jsonl\",\n max_retries: int = 50,\n retry_delay: int = 5\n\n@dataclass\nclass PromptConfig:\n \"\"\"Configuration for prompt generation\"\"\"\n\n template_dir: Optional[str] = None\n system_message: str = \"system_message\"\n evaluator_system_message: str = \"evaluator_system_message\"\n\n # Number of examples to include in the prompt\n num_top_programs: int = 3\n num_diverse_programs: int = 2\n\n # Template stochasticity\n use_template_stochasticity: bool = True\n template_variations: Dict[str, List[str]] = field(default_factory=dict)\n\n # Meta-prompting\n use_meta_prompting: bool = False\n meta_prompt_weight: float = 0.1\n\n # Artifact rendering\n include_artifacts: bool = True\n max_artifact_bytes: int = 20 * 1024 # 20KB in prompt\n artifact_security_filter: bool = True\n\n\n@dataclass\nclass DatabaseConfig:\n \"\"\"Configuration for the program database\"\"\"\n\n # General settings\n db_path: Optional[str] = None # Path to store database on disk\n in_memory: bool = True\n\n # Prompt and response logging to programs/<id>.json\n log_prompts: bool = True\n\n # Evolutionary parameters\n population_size: int = 1000\n archive_size: int = 100\n num_islands: int = 5\n\n # Selection parameters\n elite_selection_ratio: float = 0.1\n exploration_ratio: float = 0.2\n exploitation_ratio: float = 0.7\n diversity_metric: str = \"edit_distance\" # Options: \"edit_distance\", \"feature_based\"\n\n # Feature map dimensions for MAP-Elites\n feature_dimensions: List[str] = field(default_factory=lambda: [\"score\", \"complexity\"])\n feature_bins: int = 10\n\n # Migration parameters for island-based evolution\n migration_interval: int = 50 # Migrate every N generations\n migration_rate: float = 0.1 # Fraction of population to migrate\n\n # Random seed for reproducible sampling\n random_seed: Optional[int] = None\n\n # Artifact storage\n artifacts_base_path: Optional[str] = None # Defaults to db_path/artifacts\n artifact_size_threshold: int = 32 * 1024 # 32KB threshold\n cleanup_old_artifacts: bool = True\n artifact_retention_days: int = 30\n\n\n@dataclass\nclass EvaluatorConfig:\n \"\"\"Configuration for program evaluation\"\"\"\n\n # General settings\n timeout: int = 300 # Maximum evaluation time in seconds\n max_retries: int = 3\n\n # Resource limits for evaluation\n memory_limit_mb: Optional[int] = None\n cpu_limit: Optional[float] = None\n\n # Evaluation strategies\n cascade_evaluation: bool = True\n cascade_thresholds: List[float] = field(default_factory=lambda: [0.5, 0.75, 0.9])\n\n # Parallel evaluation\n parallel_evaluations: int = 4\n distributed: bool = False\n\n # LLM-based feedback\n use_llm_feedback: bool = False\n llm_feedback_weight: float = 0.1\n\n # Artifact handling\n enable_artifacts: bool = True\n max_artifact_storage: int = 100 * 1024 * 1024 # 100MB per program\n\n\n@dataclass\nclass Config:\n \"\"\"Master configuration for EvolveAgent\"\"\"\n\n # General settings\n max_iterations: int = 10000\n checkpoint_interval: int = 100\n log_level: str = \"INFO\"\n log_dir: Optional[str] = None\n random_seed: Optional[int] = None\n\n # Component configurations\n llm: LLMConfig = field(default_factory=LLMConfig)\n prompt: PromptConfig = field(default_factory=PromptConfig)\n database: DatabaseConfig = field(default_factory=DatabaseConfig)\n evaluator: EvaluatorConfig = field(default_factory=EvaluatorConfig)\n rewardmodel: RewardModelConfig = field(default_factory=RewardModelConfig)\n\n # Evolution settings\n diff_based_evolution: bool = True\n allow_full_rewrites: bool = False\n max_code_length: int = 10000\n\n @classmethod\n def from_yaml(cls, path: Union[str, Path]) -> \"Config\":\n \"\"\"Load configuration from a YAML file\"\"\"\n with open(path, \"r\") as f:\n config_dict = yaml.safe_load(f)\n return cls.from_dict(config_dict)\n\n @classmethod\n def from_dict(cls, config_dict: Dict[str, Any]) -> \"Config\":\n \"\"\"Create configuration from a dictionary\"\"\"\n # Handle nested configurations\n config = Config()\n\n # Update top-level fields\n for key, value in config_dict.items():\n if key not in [\"llm\", \"prompt\", \"database\", \"evaluator\"] and hasattr(config, key):\n setattr(config, key, value)\n\n # Update nested configs\n if \"llm\" in config_dict:\n llm_dict = config_dict[\"llm\"]\n if \"models\" in llm_dict:\n llm_dict[\"models\"] = [LLMModelConfig(**m) for m in llm_dict[\"models\"]]\n if \"evaluator_models\" in llm_dict:\n llm_dict[\"evaluator_models\"] = [\n LLMModelConfig(**m) for m in llm_dict[\"evaluator_models\"]\n ]\n config.llm = LLMConfig(**llm_dict)\n if \"prompt\" in config_dict:\n config.prompt = PromptConfig(**config_dict[\"prompt\"])\n if \"database\" in config_dict:\n config.database = DatabaseConfig(**config_dict[\"database\"])\n if \"evaluator\" in config_dict:\n config.evaluator = EvaluatorConfig(**config_dict[\"evaluator\"])\n if \"rewardmodel\" in config_dict:\n config.rewardmodel = RewardModelConfig(**config_dict[\"rewardmodel\"])\n\n return config\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Convert configuration to a dictionary\"\"\"\n return {\n # General settings\n \"max_iterations\": self.max_iterations,\n \"checkpoint_interval\": self.checkpoint_interval,\n \"log_level\": self.log_level,\n \"log_dir\": self.log_dir,\n \"random_seed\": self.random_seed,\n # Component configurations\n \"llm\": {\n \"models\": self.llm.models,\n \"evaluator_models\": self.llm.evaluator_models,\n \"api_base\": self.llm.api_base,\n \"temperature\": self.llm.temperature,\n \"top_p\": self.llm.top_p,\n \"max_tokens\": self.llm.max_tokens,\n \"timeout\": self.llm.timeout,\n \"retries\": self.llm.retries,\n \"retry_delay\": self.llm.retry_delay,\n },\n \"prompt\": {\n \"template_dir\": self.prompt.template_dir,\n \"system_message\": self.prompt.system_message,\n \"evaluator_system_message\": self.prompt.evaluator_system_message,\n \"num_top_programs\": self.prompt.num_top_programs,\n \"num_diverse_programs\": self.prompt.num_diverse_programs,\n \"use_template_stochasticity\": self.prompt.use_template_stochasticity,\n \"template_variations\": self.prompt.template_variations,\n # Note: meta-prompting features not implemented\n # \"use_meta_prompting\": self.prompt.use_meta_prompting,\n # \"meta_prompt_weight\": self.prompt.meta_prompt_weight,\n },\n \"database\": {\n \"db_path\": self.database.db_path,\n \"in_memory\": self.database.in_memory,\n \"population_size\": self.database.population_size,\n \"archive_size\": self.database.archive_size,\n \"num_islands\": self.database.num_islands,\n \"elite_selection_ratio\": self.database.elite_selection_ratio,\n \"exploration_ratio\": self.database.exploration_ratio,\n \"exploitation_ratio\": self.database.exploitation_ratio,\n # Note: diversity_metric fixed to \"edit_distance\"\n # \"diversity_metric\": self.database.diversity_metric,\n \"feature_dimensions\": self.database.feature_dimensions,\n \"feature_bins\": self.database.feature_bins,\n \"migration_interval\": self.database.migration_interval,\n \"migration_rate\": self.database.migration_rate,\n \"random_seed\": self.database.random_seed,\n \"log_prompts\": self.database.log_prompts,\n },\n \"evaluator\": {\n \"timeout\": self.evaluator.timeout,\n \"max_retries\": self.evaluator.max_retries,\n # Note: resource limits not implemented\n # \"memory_limit_mb\": self.evaluator.memory_limit_mb,\n # \"cpu_limit\": self.evaluator.cpu_limit,\n \"cascade_evaluation\": self.evaluator.cascade_evaluation,\n \"cascade_thresholds\": self.evaluator.cascade_thresholds,\n \"parallel_evaluations\": self.evaluator.parallel_evaluations,\n # Note: distributed evaluation not implemented\n # \"distributed\": self.evaluator.distributed,\n \"use_llm_feedback\": self.evaluator.use_llm_feedback,\n \"llm_feedback_weight\": self.evaluator.llm_feedback_weight,\n },\n \"rewardmodel\": {\n \"model_type\": self.rewardmodel.model_type,\n \"model\": self.rewardmodel.model,\n \"temperature\": self.rewardmodel.temperature,\n \"top_p\": self.rewardmodel.top_p,\n \"max_tokens\": self.rewardmodel.max_tokens,\n \"proposal_score_threshold\": self.rewardmodel.proposal_score_threshold,\n \"api_key\": self.rewardmodel.api_key,\n \"base_url\": self.rewardmodel.base_url,\n \"jsonl_file\": self.rewardmodel.jsonl_file,\n \"max_retries\": self.rewardmodel.max_retries,\n \"retry_delay\": self.rewardmodel.retry_delay,\n },\n # Evolution settings\n \"diff_based_evolution\": self.diff_based_evolution,\n \"allow_full_rewrites\": self.allow_full_rewrites,\n \"max_code_length\": self.max_code_length,\n }\n\n def to_yaml(self, path: Union[str, Path]) -> None:\n \"\"\"Save configuration to a YAML file\"\"\"\n with open(path, \"w\") as f:\n yaml.dump(self.to_dict(), f, default_flow_style=False)\n\n\ndef load_config(config_path: Optional[Union[str, Path]] = None) -> Config:\n \"\"\"Load configuration from a YAML file or use defaults\"\"\"\n if config_path and os.path.exists(config_path):\n config = Config.from_yaml(config_path)\n else:\n config = Config()\n\n # Use environment variables if available\n api_key = os.environ.get(\"OPENAI_API_KEY\")\n api_base = os.environ.get(\"OPENAI_API_BASE\", \"https://api.openai.com/v1\")\n\n config.llm.update_model_params({\"api_key\": api_key, \"api_base\": api_base})\n\n # Make the system message available to the individual models, in case it is not provided from the prompt sampler\n config.llm.update_model_params({\"system_message\": config.prompt.system_message})\n\n return config\n"}}, "benchmark/kissing_number/initial_program.py": {"301629169932266779eb44ff80f0169d50896f3b": {"t": 1785599211.402838, "n": 20120, "h": "15efd6f1f0af27ce", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\n592-Point Kissing Configuration - Raw Data Version\n==================================================\n\nUses raw Python data from atlas_data_raw_generated.py\nto construct 592×11D kissing configuration.\n\nAdvantages:\n- Data fully readable and editable\n- Easy to debug and modify\n- No decoding process required\n\"\"\"\n\n\"\"\"\nRaw ATLAS Data - Auto-generated\n===============================\n\nComplete vector data of PSU(4,2) group obtained from ATLAS database,\nstored in raw Python list format.\n\nData format: [[real, imag], [real, imag], ...]\nVector count: Φ₁=80, Φ₂=270\n\"\"\"\n\nimport os\nimport sys\nimport numpy as np\nfrom typing import Tuple, Dict\n\n# Try to import full ATLAS data; if unavailable, fall back to a small, provably-valid\n# default dataset so the script returns a correct (if smaller) configuration\ntry:\n from atlas_data import PHI1, PHI2 # expected shape: (n, 5) complex arrays\n ATLAS_AVAILABLE = True\n # Quick sanity validation of imported arrays\n try:\n # ensure numpy arrays and complex dtype\n PHI1 = np.asarray(PHI1, dtype=np.complex128)\n PHI2 = np.asarray(PHI2, dtype=np.complex128)\n if PHI1.ndim != 2 or PHI2.ndim != 2:\n raise ValueError(\"PHI1/PHI2 must be 2D arrays\")\n if PHI1.shape[1] != 5 or PHI2.shape[1] != 5:\n print(f\"⚠️ atlas_data found but unexpected number of columns: PHI1.shape={PHI1.shape}, PHI2.shape={PHI2.shape}; rejecting ATLAS data.\")\n ATLAS_AVAILABLE = False\n PHI1 = None\n PHI2 = None\n else:\n # Basic row count check (expected sizes from constants)\n exp_n1 = PSU42_CONSTANTS['maximal_subgroups']['H1']['frame_size']\n exp_n2 = PSU42_CONSTANTS['maximal_subgroups']['H2']['frame_size']\n if PHI1.shape[0] != exp_n1 or PHI2.shape[0] != exp_n2:\n print(f\" ⚠️ atlas_data row counts differ from expected (expected {exp_n1},{exp_n2}); continuing but will validate inner products later.\")\n except Exception as e:\n print(f\"⚠️ atlas_data import succeeded but validation failed: {e}. Rejecting ATLAS data.\")\n ATLAS_AVAILABLE = False\n PHI1 = None\n PHI2 = None\nexcept Exception:\n ATLAS_AVAILABLE = False\n # Do NOT silently fall back to brittle ad‑hoc vectors that can violate packing constraints.\n # Record absence of canonical data; later we will return a provably-valid combinatorial fallback.\n # New policy: when canonical ATLAS data are missing, construct a deterministic,\n # provably-valid combinatorial sparse configuration (greedy 4-sparse support packing)\n # that aims to exceed the current 220 two-sparse fallback while guaranteeing\n # pairwise dot ≤ 0.5 by construction (supports of size 4; entries ±1/√4).\n print(\"⚠️ atlas_data module not found or invalid — refusing silent ad-hoc fallback. Will use provable greedy 4-sparse fallback in construct_C10_C11_raw().\")\n PHI1 = None\n PHI2 = None\n\n# PSU(4,2) group constants\nPSU42_CONSTANTS = {'group_name': 'PSU(4,2)', 'atlas_name': 'U4(2)', 'group_size': 25920, 'irreducible_characters': 20, 'character_dimensions': [1, 5, 5, 6, 10, 10, 15, 15, 20, 24, 30, 30, 30, 40, 40, 45, 45, 60, 64, 81], 'dim5_positions': [2, 3], 'maximal_subgroups': {'H1': {'structure': '3^3:S4', 'size': 648, 'frame_size': 80}, 'H2': {'structure': '2.(A4:A4).2', 'size': 576, 'frame_size': 270}}}\n\ndef realify_to_R10(C5_vectors: np.ndarray) -> np.ndarray:\n \"\"\"\n ℂ^5 → ℝ^10 conversion: concatenate Re and Im parts, then renormalize.\n \"\"\"\n Re = np.real(C5_vectors)\n Im = np.imag(C5_vectors)\n X = np.concatenate([Re, Im], axis=1)\n X /= np.linalg.norm(X, axis=1, keepdims=True)\n return X\n\n\ndef construct_C10_C11_raw() -> Tuple[np.ndarray, np.ndarray]:\n \"\"\"\n Construct C10 and C11 configurations using raw ATLAS data.\n Based entirely on Ganzhinov's mathematical construction:\n C₁₁ = Π(icΦ₂) ∪ Π(Φ₁) ∪ Π(e^{2πi/3}Φ₁) ∪ (√3/2 Π(e^{4πi/3}Φ₁) ± 1/2 e₁₁) ∪ {±e₁₁}\n \"\"\"\n print(\"🔧 Constructing configuration using imported ATLAS data...\")\n\n # If canonical ATLAS data are not available, return a provably-valid fallback:\n # ±standard basis vectors in R^11 (22 points). This set has pairwise dot ∈ {0, ±1}\n # so distinct vectors have dot ≤ 0.0 ≤ 0.5 and will always pass the verifier.\n if not ATLAS_AVAILABLE or PHI1 is None or PHI2 is None:\n # New fallback: large randomized candidate pool + greedy maximum-independent-set\n # This attempts to produce a far larger valid kissing set than the conservative\n # 4-sparse fallback by exploring many random candidates and selecting a big\n # conflict-free subset (dot ≤ 0.5).\n print(\" ⚠️ ATLAS data unavailable — running randomized augmentation + MIS fallback to maximize sphere count.\")\n\n import os\n from math import sqrt\n\n # Parameters (allow override via environment for tuning)\n RNG_SEED = int(os.environ.get(\"RAND_SEED\", \"42\"))\n CANDIDATES = int(os.environ.get(\"CANDIDATE_POOL\", \"4000\")) # number of random candidates\n CAP = 0.5 + 1e-12\n DIM = 11\n\n rng = np.random.RandomState(RNG_SEED)\n\n # Start with deterministic helpful seeds: ±standard-basis (22 points)\n e = np.eye(DIM)\n seeds = np.vstack([e, -e])\n\n # Optionally include the deterministic greedy 4-sparse supports as promising structured seeds.\n # Keep support generation small to avoid explosion; use SUPPORT_W env var (default 4).\n try:\n from itertools import combinations, product\n w = int(os.environ.get(\"SUPPORT_W\", \"4\"))\n supports = list(combinations(range(DIM), w))\n # Greedy selection of supports similar to the old code but limited to first 60 supports\n selected_supports = []\n conflicts = []\n for s in supports:\n ss = set(s)\n conflicts.append({i for i, t in enumerate(supports) if len(ss & set(t)) >= 3})\n remaining = set(range(len(supports)))\n while remaining and len(selected_supports) < 60:\n best = min(remaining, key=lambda idx: (sum(1 for nb in conflicts[idx] if nb in remaining), idx))\n selected_supports.append(supports[best])\n remaining -= ({best} | conflicts[best])\n vecs = []\n for supp in selected_supports:\n for signs in product((-1.0, 1.0), repeat=w):\n v = np.zeros(DIM, dtype=float)\n for idx, si in zip(supp, signs):\n v[idx] = si / sqrt(w)\n vecs.append(v)\n if len(vecs) > 0:\n seeds = np.vstack([seeds, np.vstack(vecs)])\n except Exception:\n # If anything fails, continue with basis seeds only\n pass\n\n # Generate random candidate pool on the unit sphere\n cand = rng.normal(size=(CANDIDATES, DIM))\n cand /= np.linalg.norm(cand, axis=1, keepdims=True)\n\n # Combine seeds and candidates (seeds first to bias selection towards them)\n X_all = np.vstack([seeds, cand])\n n_total = X_all.shape[0]\n\n # Build conflict boolean adjacency (True if conflict: dot > CAP)\n dots_chunk = X_all @ X_all.T\n adj = dots_chunk > CAP\n np.fill_diagonal(adj, False)\n\n # Greedy maximum independent set: repeatedly pick vertex with minimal degree\n remaining = set(range(n_total))\n degrees = np.sum(adj, axis=1)\n indep = []\n # Precompute neighbor sets for speed\n neighbor_sets = [set(np.nonzero(adj[i])[0]) for i in range(n_total)]\n\n while remaining:\n # pick vertex in remaining with minimal current degree (ties by index)\n # compute dynamic degrees restricted to 'remaining'\n best = min(remaining, key=lambda i: (sum(1 for nb in neighbor_sets[i] if nb in remaining), i))\n indep.append(best)\n # remove best and its neighbors\n to_remove = {best} | (neighbor_sets[best] & remaining)\n remaining -= to_remove\n\n sphere_centers = X_all[indep]\n # Defensive normalization\n sphere_centers = sphere_centers / np.linalg.norm(sphere_centers, axis=1, keepdims=True)\n\n # Small repulsion relaxation (tangent-space smoothing) to try and reduce near-violations\n def relax_positions(Y, cap=CAP, iters=60, eps=0.02):\n Y = Y.copy()\n for _ in range(iters):\n D = Y @ Y.T\n np.fill_diagonal(D, -np.inf)\n # Find all violating pairs\n viol_i, viol_j = np.where(D > cap)\n if len(viol_i) == 0:\n break\n # Accumulate repulsive displacement per vector\n disp = np.zeros_like(Y)\n for i, j in zip(viol_i, viol_j):\n # vector pointing away from neighbor in ambient space\n v = Y[i] - Y[j]\n # scale by how bad the violation is\n w = (D[i, j] - cap)\n if np.linalg.norm(v) > 0:\n disp[i] += (w * v) / (np.linalg.norm(v) + 1e-12)\n # apply displacements and renormalize\n Y += eps * disp\n Y /= np.linalg.norm(Y, axis=1, keepdims=True)\n return Y\n\n sphere_centers = relax_positions(sphere_centers, iters=80, eps=0.015)\n\n # Final cleanup: if any violations remain, greedily drop worst offenders until valid\n def greedy_prune(Y, cap=CAP):\n Y = Y.copy()\n if Y.shape[0] <= 1:\n return Y\n for _ in range(max(1, Y.shape[0] // 2)):\n D = Y @ Y.T\n np.fill_diagonal(D, -np.inf)\n max_dot = float(np.max(D))\n if max_dot <= cap:\n break\n # remove one of the pair involved in worst violation: pick the one with larger total violations\n flat = int(np.argmax(D))\n m = D.shape[1]\n i = flat // m\n j = flat % m\n pos_counts = np.sum(D > cap, axis=1)\n if pos_counts[i] > pos_counts[j]:\n remove_idx = i\n else:\n remove_idx = j\n mask = np.ones(Y.shape[0], dtype=bool)\n mask[remove_idx] = False\n Y = Y[mask]\n # final assert-free check: if still invalid, fall back to a conservative set\n D = Y @ Y.T\n np.fill_diagonal(D, -np.inf)\n if np.any(D > cap):\n # fallback to ±standard-basis to avoid returning invalid config\n e11 = np.eye(DIM)\n return np.vstack([e11, -e11])\n return Y\n\n sphere_centers = greedy_prune(sphere_centers)\n\n # Final defensive normalization and validation\n sphere_centers = sphere_centers / np.linalg.norm(sphere_centers, axis=1, keepdims=True)\n D = sphere_centers @ sphere_centers.T\n np.fill_diagonal(D, -np.inf)\n max_dot = float(np.max(D)) if sphere_centers.shape[0] > 1 else -1.0\n print(f\" ℹ️ Randomized MIS fallback produced {sphere_centers.shape[0]} vectors; max_dot={max_dot:.6g}\")\n\n # Return empty C10 (no 10D construction) and our randomized fallback\n return np.empty((0, 10)), sphere_centers\n\n # Phase choices\n omega = np.exp(2j * np.pi / 3)\n i_times_Phi2 = 1j * PHI2\n P1 = PHI1\n P2 = omega * PHI1\n P3 = (omega ** 2) * PHI1\n\n # Construct 10D code\n print(\" Constructing 10D configuration C₁₀...\")\n C10_complex = np.vstack([i_times_Phi2, P1, P2, P3])\n C10 = realify_to_R10(C10_complex)\n \n n1 = PHI1.shape[0]\n n2 = PHI2.shape[0]\n expected_c10_rows = n2 + 3 * n1\n \n assert C10.shape[0] == expected_c10_rows and C10.shape[1] == 10, f\"Unexpected C10 shape. Got {C10.shape}, expected ({expected_c10_rows}, 10)\"\n\n # Lift to 11D\n print(\" Lifting to 11D configuration C₁₁...\")\n def embed(v10):\n return np.hstack([v10, np.zeros((v10.shape[0], 1))])\n\n # Split by family sizes\n Pi_iPhi2 = embed(C10[:n2])\n Pi_P1 = embed(C10[n2:n2 + n1])\n Pi_P2 = embed(C10[n2 + n1:n2 + 2 * n1])\n Pi_P3 = embed(C10[n2 + 2 * n1:])\n\n e11 = np.zeros((1, 11)); e11[0, 10] = 1.0\n shifted_plus = (np.sqrt(3) / 2) * Pi_P3 + 0.5 * e11\n shifted_minus = (np.sqrt(3) / 2) * Pi_P3 - 0.5 * e11\n\n sphere_centers = np.vstack([\n Pi_iPhi2,\n Pi_P1,\n Pi_P2,\n shifted_plus,\n shifted_minus,\n e11,\n -e11\n ])\n \n expected_c11_rows = n2 + 4 * n1 + 2\n assert sphere_centers.shape[0] == expected_c11_rows and sphere_centers.shape[1] == 11, f\"Unexpected C11 shape. Got {sphere_centers.shape}, expected ({expected_c11_rows}, 11)\"\n \n # Ensure all vectors are unit vectors\n sphere_centers = sphere_centers / np.linalg.norm(sphere_centers, axis=1, keepdims=True)\n\n # Optionally enforce dot product cap (<= 0.5) by greedy removal\n def enforce_dot_cap(X: np.ndarray, cap: float = 0.5 + 1e-9, max_removals: int = 1000) -> np.ndarray:\n \"\"\"\n Enforce dot(X[i], X[j]) <= cap for all distinct i,j by greedy removals.\n Behavior changes:\n - Allow more removals (default up to 1000 or up to len(X)-2).\n - After removals, if any violation remains, raise RuntimeError so caller can fallback.\n - Returns the pruned array on success.\n \"\"\"\n removed_count = 0\n # Defensive: if X is empty or trivial, return immediately\n if X.shape[0] <= 1:\n return X\n\n # Cap max_removals also by number of vectors\n max_allowed = min(max_removals, max(0, X.shape[0] - 2))\n\n for _ in range(max_allowed):\n dot_products = X @ X.T # shape (m, m)\n # exclude diagonal from consideration by setting to -inf\n np.fill_diagonal(dot_products, -np.inf)\n\n # identify positive violations only\n if not np.any(dot_products > cap):\n break\n\n # index of largest dot value (worst positive violation)\n flat_idx = int(np.argmax(dot_products))\n m = dot_products.shape[1]\n i = flat_idx // m\n j = flat_idx % m\n # defensive: if somehow (-inf) chosen, break\n if dot_products[i, j] == -np.inf:\n break\n\n # Count positive violations per vector (how many other vectors it has dot>cap)\n pos_counts = np.sum(dot_products > cap, axis=1)\n\n # Choose which of i or j to remove:\n # prefer removing the one participating in more positive violations;\n # if tie, remove the one whose maximum offending dot is larger.\n if pos_counts[i] > pos_counts[j]:\n remove_idx = i\n elif pos_counts[j] > pos_counts[i]:\n remove_idx = j\n else:\n # tie-breaker: remove the one with larger max dot entry\n if np.max(dot_products[i]) >= np.max(dot_products[j]):\n remove_idx = i\n else:\n remove_idx = j\n\n # Perform removal\n mask = np.ones(X.shape[0], dtype=bool)\n mask[remove_idx] = False\n X = X[mask]\n removed_count += 1\n\n # If too few vectors remain, break early\n if X.shape[0] <= 1:\n break\n\n # Final validation: ensure no violations remain\n if X.shape[0] > 1:\n dot_products = X @ X.T\n np.fill_diagonal(dot_products, -np.inf)\n if np.any(dot_products > cap):\n # Collect a small diagnostic top violation\n max_dot = float(np.max(dot_products))\n flat_idx = int(np.argmax(dot_products))\n m = dot_products.shape[1]\n i = flat_idx // m\n j = flat_idx % m\n raise RuntimeError(f\"enforce_dot_cap: unable to reach cap={cap:.12g}; max_dot={max_dot:.12g} at pair ({i},{j}); removed={removed_count}\")\n\n if removed_count > 0:\n print(f\" ⚠️ Removed {removed_count} vectors to enforce dot≤0.5. New count: {X.shape[0]}\")\n return X\n\n try:\n sphere_centers = enforce_dot_cap(sphere_centers, max_removals=1000)\n except RuntimeError as e:\n # If greedy repair fails, do not return an invalid large configuration.\n # Save diagnostics and fall back to provable ±standard basis in R11.\n try:\n import tempfile, time\n fn = os.path.join(tempfile.gettempdir(), f\"atlas_bad_{int(time.time())}.npz\")\n # Save offending centers for offline analysis (if available)\n np.savez_compressed(fn, sphere_centers_raw=sphere_centers)\n print(f\" ⚠️ enforce_dot_cap failed: {e}. Saved diagnostic npz to {fn}\")\n except Exception:\n print(f\" ⚠️ enforce_dot_cap failed: {e}. Failed to write diagnostic npz.\")\n print(\" ❌ Falling back to provable ±standard-basis in R11 (22 points).\")\n e11 = np.eye(11)\n sphere_centers = np.vstack([e11, -e11])\n return C10, sphere_centers\n\n print(f\" ✅ C₁₀: {C10.shape} ({C10.shape[0]}×10D configuration)\")\n print(f\" ✅ C₁₁: {sphere_centers.shape} ({sphere_centers.shape[0]}×11D configuration)\")\n\n return C10, sphere_centers\n\n\ndef analyze_raw_data_quality():\n \"\"\"Analyze the quality of raw data\"\"\"\n print(\"\\n📊 Analyzing raw data quality...\")\n \n # If PHI1/PHI2 are not present (missing atlas data), skip analysis.\n if PHI1 is None or PHI2 is None:\n print(\" ⚠️ PHI1/PHI2 not available; skipping raw data quality analysis.\")\n return\n\n # Check vector norms\n phi1_norms = np.linalg.norm(PHI1, axis=1)\n phi2_norms = np.linalg.norm(PHI2, axis=1)\n \n print(f\" Φ₁ norm range: [{np.min(phi1_norms):.6f}, {np.max(phi1_norms):.6f}]\")\n print(f\" Φ₂ norm range: [{np.min(phi2_norms):.6f}, {np.max(phi2_norms):.6f}]\")\n \n # Check if they are unit vectors\n phi1_unit = np.allclose(phi1_norms, 1.0, atol=1e-10)\n phi2_unit = np.allclose(phi2_norms, 1.0, atol=1e-10)\n \n print(f\" Φ₁ unit vectors: {'✅' if phi1_unit else '❌'}\")\n print(f\" Φ₂ unit vectors: {'✅' if phi2_unit else '❌'}\")\n \n # Check inner product distribution\n phi1_inner_products = PHI1 @ PHI1.T.conj()\n phi2_inner_products = PHI2 @ PHI2.T.conj()\n \n # Remove diagonal elements\n np.fill_diagonal(phi1_inner_products, 0)\n np.fill_diagonal(phi2_inner_products, 0)\n \n phi1_max_ip = np.max(np.abs(phi1_inner_products))\n phi2_max_ip = np.max(np.abs(phi2_inner_products))\n \n print(f\" Φ₁ max inner product: {phi1_max_ip:.6f}\")\n print(f\" Φ₂ max inner product: {phi2_max_ip:.6f}\")\n\n\ndef main():\n \"\"\"Main function, construct C11 configuration using raw ATLAS data\"\"\"\n print(\"=== 11D Kissing Number: Raw Data Version K(11) ≥ 592 ===\")\n \n # Analyze raw data quality (skip gracefully under exec with split globals/locals)\n try: # Some evaluators exec with split globals/locals causing NameError\n analyze_raw_data_quality()\n except NameError:\n pass\n \n # Construct C10 and C11\n C10, sphere_centers = construct_C10_C11_raw()\n\n if sphere_centers is None:\n print(\"❌ Failed to construct C11 configuration.\")\n return None\n\n print(f\"\\n✅ Successfully constructed C11 configuration, shape: {sphere_centers.shape}\")\n\n return sphere_centers\n\n\nif __name__ == \"__main__\":\n sphere_centers = main()\n\n# Ensure compatibility with exec/import-based evaluators that expect a global\n# `sphere_centers` without running as __main__\ntry: # noqa: SIM105\n sphere_centers # type: ignore[name-defined]\nexcept NameError:\n # Make function definitions visible in environments where exec() is called\n # with separate globals/locals mappings (functions may land in locals)\n try:\n globals().update(locals())\n except Exception:\n pass\n sphere_centers = main()\n"}}, "benchmark/MSTD/initial_program.py": {"56af1bdc92f66a7d5c068db8d9b04b6fab306622": {"t": 1785599211.4220548, "n": 436, "h": "2b8ac0f7e89f75eb", "c": "import numpy as np\n\n\ndef main():\n N = 30\n # Conway MSTD set example; we take A=B for classical MSTD\n A = [0, 2, 3, 4, 7, 11, 12, 14]\n B = A[:]\n A_ind = np.zeros(N, dtype=int); A_ind[A] = 1\n B_ind = np.zeros(N, dtype=int); B_ind[B] = 1\n return A_ind, B_ind\n\n\n# Ensure globals for evaluator\ntry:\n A_indicators; B_indicators # type: ignore[name-defined]\nexcept NameError:\n A_indicators, B_indicators = main()\n\n\n"}}, "benchmark/third_autocorrelation_inequality/evaluator.py": {"11ac0e4c2c98aea128f7feea19cbc90cdd6e9854": {"t": 1785599212.1877966, "n": 938, "h": "c73300cab5627bf1", "c": "import numpy as np\n\ndef evaluate(program_path: str = \"/data/zhuotaodeng/yzj/_para/alpha-research/math/initial_program.py\"):\n \"\"\"\n Evaluate the pack_circles function from the given program file.\n Returns the total radius sum if valid, otherwise raises an exception.\n \"\"\"\n import importlib.util\n import sys\n \n # Load the module from the given path\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n try:\n height_sequence_3 = program.find_better_c3_upper_bound()\n except:\n return {\"error\": -10.0}\n \n convolution_3 = np.convolve(height_sequence_3, height_sequence_3)\n C_upper_bound = abs(2 * len(height_sequence_3) * np.max(convolution_3) / (np.sum(height_sequence_3)**2))\n \n return {\"score\": 1.0 / C_upper_bound}\n\nprint(evaluate())"}}, "benchmark/spherical_code/initial_program.py": {"122a43624230c97ef2e50a70bbef6962c9ed0a01": {"t": 1785599212.2244709, "n": 2502, "h": "aa964098278d9f8d", "c": "import numpy as np\n\ndef _normalize_rows(P):\n nrm = np.linalg.norm(P, axis=1, keepdims=True)\n nrm = np.maximum(nrm, 1e-12)\n return P / nrm\n\ndef seed_platonic(n):\n \"\"\"Return a good symmetric seed on S^2 for some n; else None.\"\"\"\n if n == 2: # antipodal\n return np.array([[0,0,1],[0,0,-1]], dtype=float)\n if n == 3: # equilateral on equator\n ang = 2*np.pi/3\n return np.array([[1,0,0],[np.cos(ang),np.sin(ang),0],[np.cos(2*ang),np.sin(2*ang),0]], dtype=float)\n if n == 4: # tetrahedron\n return _normalize_rows(np.array([[1,1,1],[1,-1,-1],[-1,1,-1],[-1,-1,1]], dtype=float))\n if n == 6: # octahedron\n return np.array([[1,0,0],[-1,0,0],[0,1,0],[0,-1,0],[0,0,1],[0,0,-1]], dtype=float)\n if n == 8: # cube vertices\n V = np.array([[sx,sy,sz] for sx in (-1,1) for sy in (-1,1) for sz in (-1,1)], dtype=float)\n return _normalize_rows(V)\n if n == 12: # icosahedron (one realization)\n phi = (1+np.sqrt(5))/2\n V = []\n for s in (-1,1):\n V += [[0, s, phi],[0, s, -phi],[ s, phi,0],[ s, -phi,0],[ phi,0, s],[-phi,0, s]]\n V = np.array(V, dtype=float)\n return _normalize_rows(V)\n return None\n\ndef farthest_point_greedy(n, seed=None, rng=np.random.default_rng(0)):\n \"\"\"Greedy max–min on S^2: start from seed (if any), then add points that maximize min angle.\"\"\"\n def random_unit(k):\n X = rng.normal(size=(k,3)); return _normalize_rows(X)\n\n if seed is None:\n P = random_unit(1) # start with one random point\n else:\n P = _normalize_rows(seed)\n while len(P) < n:\n # generate candidates and pick the one with largest min angle to current set\n C = random_unit(2000) # candidates per iteration (tune as needed)\n # cosines to existing points\n cos = C @ P.T\n # min angle to set -> maximize this\n min_ang = np.arccos(np.clip(np.max(cos, axis=1), -1.0, 1.0))\n idx = np.argmax(min_ang)\n P = np.vstack([P, C[idx:idx+1]])\n return P\n\ndef main():\n n = 30\n seed = seed_platonic(n)\n pts = farthest_point_greedy(n, seed=seed, rng=np.random.default_rng(42))\n print(f\"n={n}, points={len(pts)}\")\n return pts\n\nif __name__ == \"__main__\":\n points = main()\n # 保存为 npy 文件\n np.save(\"points.npy\", points)\n print(\"已保存 points.npy\")\n\n# Ensure compatibility with evaluators that expect a global variable\ntry:\n points # type: ignore[name-defined]\nexcept NameError:\n points = main()\n"}}, "evolve_agent/__init__.py": {"313177eecc1cbab350d924f3d433abfa181835b9": {"t": 1785599212.3929868, "n": 98, "h": "c2e4575ded444420", "c": "__version__ = \"0.1.0\"\n\nfrom evolve_agent.controller import EvolveAgent\n\n__all__ = [\"EvolveAgent\"]\n"}}, "benchmark/spherical_code/evaluator.py": {"fb870bca7febfaa817b9a2f54255af845099346f": {"t": 1785599212.4044003, "n": 2096, "h": "ea9cc3cdd6029604", "c": "import sys\nimport os\nimport json\nimport numpy as np\nfrom typing import Dict\nimport importlib.util\n\nEPS = 1e-12\n\ndef evaluate_spherical_code_min_angle(points: np.ndarray) -> Dict[str, float]:\n P = np.asarray(points, dtype=float)\n if P.ndim != 2 or P.shape[0] < 2:\n return {\"valid\": 0.0, \"min_angle\": 0.0, \"n\": 0.0, \"dimension\": 0.0, \"score\": 0.0}\n # normalize rows onto the sphere\n norms = np.maximum(np.linalg.norm(P, axis=1, keepdims=True), EPS)\n P = P / norms\n n = P.shape[0]\n d = P.shape[1]\n min_angle = float(\"inf\")\n for i in range(n):\n for j in range(i+1, n):\n cosang = float(np.clip(np.dot(P[i], P[j]), -1.0, 1.0))\n ang = float(np.arccos(cosang))\n if ang < min_angle:\n min_angle = ang\n return {\"valid\": 1.0, \"min_angle\": float(min_angle), \"n\": float(n), \"dimension\": float(d), \"score\": float(min_angle)}\n\ndef evaluate(program_path: str):\n try:\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n\n pts = None\n if hasattr(program, 'points'):\n pts = program.points\n elif hasattr(program, 'main'):\n res = program.main()\n if isinstance(res, np.ndarray):\n pts = res\n elif hasattr(program, 'points'):\n pts = program.points\n if pts is None:\n return {\"error\": -1.0}\n result = evaluate_spherical_code_min_angle(pts)\n return {\"score\": result[\"score\"], \"min_angle\": result[\"min_angle\"], \"n\": result[\"n\"], \"dimension\": result[\"dimension\"]}\n except Exception:\n return {\"error\": -1.0}\n\nif __name__ == \"__main__\":\n try:\n default_path = os.path.join(os.path.dirname(__file__), \"initial_program.py\")\n except Exception:\n default_path = \"initial_program.py\"\n target = sys.argv[1] if len(sys.argv) > 1 else default_path\n print(json.dumps(evaluate(target), ensure_ascii=False, indent=2))\n"}}, "benchmark/packing_circles/evaluator.py": {"b3b1269a935a24e1f0a95756e5a97330e7f96f3f": {"t": 1785599212.426435, "n": 2265, "h": "e5fd787c8c219c87", "c": "import math\nimport numpy as np\nimport itertools\nimport random\n\nrandom.seed(42)\nnp.random.seed(42)\n\n\ndef verify_circles(circles):\n \"\"\"Checks that the circles are disjoint and lie inside a unit square.\n\n Args:\n circles: A list of tuples (x, y, radius) or numpy array of shape (num_circles, 3)\n\n Returns:\n bool: True if circles are valid (disjoint and within unit square), False otherwise\n \"\"\"\n # Convert to numpy array if it's a list\n if not isinstance(circles, np.ndarray):\n circles = np.array(circles)\n \n # Check pairwise disjointness\n for circle1, circle2 in itertools.combinations(circles, 2):\n center_distance = np.sqrt((circle1[0] - circle2[0])**2 + (circle1[1] - circle2[1])**2)\n radii_sum = circle1[2] + circle2[2]\n if center_distance < radii_sum:\n return False\n\n # Check all circles lie inside the unit square [0,1]x[0,1]\n for circle in circles:\n if not (0 <= min(circle[0], circle[1]) - circle[2] and max(circle[0], circle[1]) + circle[2] <= 1):\n return False\n \n return True\n\n\ndef evaluate(program_path: str = \"results/initial_program.py\"):\n \"\"\"\n Evaluate the pack_circles function from the given program file.\n Returns dict with keys: score, result_26, result_32; score is sum of totals.\n \"\"\"\n import importlib.util\n import sys\n \n # Load the module from the given path\n spec = importlib.util.spec_from_file_location(\"program\", program_path)\n program = importlib.util.module_from_spec(spec)\n sys.modules[\"program\"] = program\n spec.loader.exec_module(program)\n \n # Test the pack_circles function\n try:\n total_r_26, circles_26 = program.pack_circles(26)\n total_r_32, circles_32 = program.pack_circles(32)\n except Exception as e:\n return {\"error\": -10.0}\n \n # Validate the circles\n valid_26 = verify_circles(circles_26)\n valid_32 = verify_circles(circles_32)\n\n if not all((valid_26, valid_32)):\n return {\"error\": -1.0}\n \n score = float(total_r_26 + total_r_32)\n return {\n \"score\": score,\n \"result_26\": total_r_26,\n \"result_32\": total_r_32\n }\n\nprint(evaluate(\"/data/zhuotaodeng/yzj/_para/alpha-research/results_circles_1/test.py\"))"}}, "evolve_agent/cli.py": {"73dabc0781eb272d3e8d0cfe3208d0b140a67546": {"t": 1785599212.5882688, "n": 5553, "h": "b3fc673d97b7c57c", "c": "\"\"\"\nCommand-line interface for EvolveAgent\n\"\"\"\n\nimport argparse\nimport asyncio\nimport logging\nimport os\nimport sys\nfrom typing import Dict, List, Optional\n\nfrom evolve_agent import EvolveAgent\nfrom evolve_agent.config import Config, load_config\n\nlogger = logging.getLogger(__name__)\n\n\ndef parse_args() -> argparse.Namespace:\n \"\"\"Parse command-line arguments\"\"\"\n parser = argparse.ArgumentParser(description=\"EvolveAgent - Evolutionary coding agent\")\n\n parser.add_argument(\"initial_program\", help=\"Path to the initial program file\")\n\n parser.add_argument(\n \"evaluation_file\", help=\"Path to the evaluation file containing an 'evaluate' function\"\n )\n\n parser.add_argument(\"--config\", \"-c\", help=\"Path to configuration file (YAML)\", default=None)\n\n parser.add_argument(\"--output\", \"-o\", help=\"Output directory for results\", default=None)\n\n parser.add_argument(\n \"--iterations\", \"-i\", help=\"Maximum number of iterations\", type=int, default=None\n )\n\n parser.add_argument(\n \"--target-score\", \"-t\", help=\"Target score to reach\", type=float, default=None\n )\n\n parser.add_argument(\n \"--log-level\",\n \"-l\",\n help=\"Logging level\",\n choices=[\"DEBUG\", \"INFO\", \"WARNING\", \"ERROR\", \"CRITICAL\"],\n default=\"INFO\",\n )\n\n parser.add_argument(\n \"--checkpoint\",\n help=\"Path to checkpoint directory to resume from (e.g., evolve_agent_output/checkpoints/checkpoint_50)\",\n default=None,\n )\n\n parser.add_argument(\"--api-base\", help=\"Base URL for the LLM API\", default=None)\n\n parser.add_argument(\"--primary-model\", help=\"Primary LLM model name\", default=None)\n\n parser.add_argument(\"--secondary-model\", help=\"Secondary LLM model name\", default=None)\n\n return parser.parse_args()\n\n\nasync def main_async() -> int:\n \"\"\"\n Main asynchronous entry point\n\n Returns:\n Exit code\n \"\"\"\n args = parse_args()\n\n # Check if files exist\n if not os.path.exists(args.initial_program):\n print(f\"Error: Initial program file '{args.initial_program}' not found\")\n return 1\n\n if not os.path.exists(args.evaluation_file):\n print(f\"Error: Evaluation file '{args.evaluation_file}' not found\")\n return 1\n\n # Create config object with command-line overrides\n config = None\n if args.api_base or args.primary_model or args.secondary_model:\n # Load base config from file or defaults\n config = load_config(args.config)\n\n # Apply command-line overrides\n if args.api_base:\n config.llm.api_base = args.api_base\n print(f\"Using API base: {config.llm.api_base}\")\n\n if args.primary_model:\n config.llm.primary_model = args.primary_model\n print(f\"Using primary model: {config.llm.primary_model}\")\n\n if args.secondary_model:\n config.llm.secondary_model = args.secondary_model\n print(f\"Using secondary model: {config.llm.secondary_model}\")\n\n # Initialize EvolveAgent\n try:\n evolve_agent = EvolveAgent(\n initial_program_path=args.initial_program,\n evaluation_file=args.evaluation_file,\n config=config,\n config_path=args.config if config is None else None,\n output_dir=args.output,\n )\n\n # Load from checkpoint if specified\n if args.checkpoint:\n if not os.path.exists(args.checkpoint):\n print(f\"Error: Checkpoint directory '{args.checkpoint}' not found\")\n return 1\n print(f\"Loading checkpoint from {args.checkpoint}\")\n evolve_agent.database.load(args.checkpoint)\n print(\n f\"Checkpoint loaded successfully (iteration {evolve_agent.database.last_iteration})\"\n )\n\n # Override log level if specified\n if args.log_level:\n logging.getLogger().setLevel(getattr(logging, args.log_level))\n\n # Run evolution\n best_program = await evolve_agent.run(\n iterations=args.iterations,\n target_score=args.target_score,\n )\n\n # Get the checkpoint path\n checkpoint_dir = os.path.join(evolve_agent.output_dir, \"checkpoints\")\n latest_checkpoint = None\n if os.path.exists(checkpoint_dir):\n checkpoints = [\n os.path.join(checkpoint_dir, d)\n for d in os.listdir(checkpoint_dir)\n if os.path.isdir(os.path.join(checkpoint_dir, d))\n ]\n if checkpoints:\n latest_checkpoint = sorted(\n checkpoints, key=lambda x: int(x.split(\"_\")[-1]) if \"_\" in x else 0\n )[-1]\n\n print(f\"\\nEvolution complete!\")\n print(f\"Best program metrics:\")\n for name, value in best_program.metrics.items():\n # Handle mixed types: format numbers as floats, others as strings\n if isinstance(value, (int, float)):\n print(f\" {name}: {value:.4f}\")\n else:\n print(f\" {name}: {value}\")\n\n if latest_checkpoint:\n print(f\"\\nLatest checkpoint saved at: {latest_checkpoint}\")\n print(f\"To resume, use: --checkpoint {latest_checkpoint}\")\n\n return 0\n\n except Exception as e:\n print(f\"Error: {str(e)}\")\n import traceback\n\n traceback.print_exc()\n return 1\n\n\ndef main() -> int:\n \"\"\"\n Main entry point\n\n Returns:\n Exit code\n \"\"\"\n return asyncio.run(main_async())\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n"}}, "evolve_agent/llm/__init__.py": {"0b6f17bd1b0f8fa0dad71a968f0684ee7da83b58": {"t": 1785599212.6521196, "n": 234, "h": "d59794a27ca5fd2f", "c": "\"\"\"\nLLM module initialization\n\"\"\"\n\nfrom evolve_agent.llm.base import LLMInterface\nfrom evolve_agent.llm.ensemble import LLMEnsemble\nfrom evolve_agent.llm.openai import OpenAILLM\n\n__all__ = [\"LLMInterface\", \"OpenAILLM\", \"LLMEnsemble\"]\n"}}, "evolve_agent/database.py": {"e85afb9d465816d0a6d3781ee338201db557c5cb": {"t": 1785599212.8334584, "n": 54050, "h": "e86295179ee81cee", "c": "\"\"\"\nProgram database for EvolveAgent\n\"\"\"\n\nimport base64\nimport json\nimport logging\nimport os\nimport random\nimport time\nfrom dataclasses import asdict, dataclass, field, fields\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Set, Tuple, Union\n\nimport numpy as np\n\nfrom evolve_agent.config import DatabaseConfig\nfrom evolve_agent.utils.code_utils import calculate_edit_distance\nfrom evolve_agent.utils.metrics_utils import safe_numeric_average\n\nlogger = logging.getLogger(__name__)\n\n\ndef _safe_sum_metrics(metrics: Dict[str, Any]) -> float:\n \"\"\"Safely sum only numeric metric values, ignoring strings and other types\"\"\"\n numeric_values = [\n v for v in metrics.values() if isinstance(v, (int, float)) and not isinstance(v, bool)\n ]\n return sum(numeric_values) if numeric_values else 0.0\n\n\ndef _safe_avg_metrics(metrics: Dict[str, Any]) -> float:\n \"\"\"Safely calculate average of only numeric metric values\"\"\"\n numeric_values = [\n v for v in metrics.values() if isinstance(v, (int, float)) and not isinstance(v, bool)\n ]\n return sum(numeric_values) / max(1, len(numeric_values)) if numeric_values else 0.0\n\n\n@dataclass\nclass Program:\n \"\"\"Represents a program in the database\"\"\"\n\n # Program identification\n id: str\n code: str\n language: str = \"python\"\n\n # Proposal\n proposal: List[str] = None\n idea_reward: List[dict] = None\n\n # Evolution information\n parent_id: Optional[str] = None\n generation: int = 0\n timestamp: float = field(default_factory=time.time)\n iteration_found: int = 0 # Track which iteration this program was found\n\n # Performance metrics\n metrics: Dict[str, float] = field(default_factory=dict)\n\n # Derived features\n complexity: float = 0.0\n diversity: float = 0.0\n\n # Metadata\n metadata: Dict[str, Any] = field(default_factory=dict)\n\n # Artifact storage\n artifacts_json: Optional[str] = None # JSON-serialized small artifacts\n artifact_dir: Optional[str] = None # Path to large artifact files\n\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Convert to dictionary representation\"\"\"\n return asdict(self)\n\n @classmethod\n def from_dict(cls, data: Dict[str, Any]) -> \"Program\":\n \"\"\"Create from dictionary representation\"\"\"\n # Get the valid field names for the Program dataclass\n valid_fields = {f.name for f in fields(cls)}\n\n # Filter the data to only include valid fields\n filtered_data = {k: v for k, v in data.items() if k in valid_fields}\n\n # Log if we're filtering out any fields\n if len(filtered_data) != len(data):\n filtered_out = set(data.keys()) - set(filtered_data.keys())\n logger.debug(f\"Filtered out unsupported fields when loading Program: {filtered_out}\")\n\n return cls(**filtered_data)\n\n\nclass ProgramDatabase:\n \"\"\"\n Database for storing and sampling programs during evolution\n\n The database implements a combination of MAP-Elites algorithm and\n island-based population model to maintain diversity during evolution.\n It also tracks the absolute best program separately to ensure it's never lost.\n \"\"\"\n\n def __init__(self, config: DatabaseConfig):\n self.config = config\n\n # In-memory program storage\n self.programs: Dict[str, Program] = {}\n\n # Feature grid for MAP-Elites\n self.feature_map: Dict[str, str] = {}\n self.feature_bins = config.feature_bins\n\n # Island populations\n self.islands: List[Set[str]] = [set() for _ in range(config.num_islands)]\n\n # Island management attributes\n self.current_island: int = 0\n self.island_generations: List[int] = [0] * config.num_islands\n self.last_migration_generation: int = 0\n self.migration_interval: int = getattr(config, \"migration_interval\", 10) # Default to 10\n self.migration_rate: float = getattr(config, \"migration_rate\", 0.1) # Default to 0.1\n\n # Archive of elite programs\n self.archive: Set[str] = set()\n\n # Track the absolute best program separately\n self.best_program_id: Optional[str] = None\n\n # Track the last iteration number (for resuming)\n self.last_iteration: int = 0\n\n # Load database from disk if path is provided\n if config.db_path and os.path.exists(config.db_path):\n self.load(config.db_path)\n\n # Prompt log\n self.prompts_by_program: Dict[str, Dict[str, Dict[str, str]]] = None\n\n # Set random seed for reproducible sampling if specified\n if config.random_seed is not None:\n import random\n\n random.seed(config.random_seed)\n logger.debug(f\"Database: Set random seed to {config.random_seed}\")\n\n logger.info(f\"Initialized program database with {len(self.programs)} programs\")\n\n def add(\n self, program: Program, iteration: int = None, target_island: Optional[int] = None\n ) -> str:\n \"\"\"\n Add a program to the database\n\n Args:\n program: Program to add\n iteration: Current iteration (defaults to last_iteration)\n target_island: Specific island to add to (uses current_island if None)\n\n Returns:\n Program ID\n \"\"\"\n # Store the program\n # If iteration is provided, update the program's iteration_found\n if iteration is not None:\n program.iteration_found = iteration\n # Update last_iteration if needed\n self.last_iteration = max(self.last_iteration, iteration)\n\n self.programs[program.id] = program\n\n # Enforce population size limit\n self._enforce_population_limit()\n\n # Calculate feature coordinates for MAP-Elites\n feature_coords = self._calculate_feature_coords(program)\n\n # Add to feature map (replacing existing if better)\n feature_key = self._feature_coords_to_key(feature_coords)\n should_replace = feature_key not in self.feature_map\n\n if not should_replace:\n # Check if the existing program still exists before comparing\n existing_program_id = self.feature_map[feature_key]\n if existing_program_id not in self.programs:\n # Stale reference, replace it\n should_replace = True\n logger.debug(\n f\"Replacing stale program reference {existing_program_id} in feature map\"\n )\n else:\n # Program exists, compare fitness\n should_replace = self._is_better(program, self.programs[existing_program_id])\n\n if should_replace:\n self.feature_map[feature_key] = program.id\n\n # Add to specific island (not random!)\n island_idx = target_island if target_island is not None else self.current_island\n island_idx = island_idx % len(self.islands) # Ensure valid island\n self.islands[island_idx].add(program.id)\n\n # Track which island this program belongs to\n program.metadata[\"island\"] = island_idx\n\n # Update archive\n self._update_archive(program)\n\n # Update the absolute best program tracking\n self._update_best_program(program)\n\n # Save to disk if configured\n if self.config.db_path:\n self._save_program(program)\n\n logger.debug(f\"Added program {program.id} to island {island_idx}\")\n return program.id\n\n def get(self, program_id: str) -> Optional[Program]:\n \"\"\"\n Get a program by ID\n\n Args:\n program_id: Program ID\n\n Returns:\n Program or None if not found\n \"\"\"\n return self.programs.get(program_id)\n\n def sample(self) -> Tuple[Program, List[Program]]:\n \"\"\"\n Sample a program and inspirations for the next evolution step\n\n Returns:\n Tuple of (parent_program, inspiration_programs)\n \"\"\"\n # Select parent program\n parent = self._sample_parent()\n\n # Select inspirations\n inspirations = self._sample_inspirations(parent, n=5)\n\n logger.debug(f\"Sampled parent {parent.id} and {len(inspirations)} inspirations\")\n return parent, inspirations\n\n def get_best_program(self, metric: Optional[str] = None) -> Optional[Program]:\n \"\"\"\n Get the best program based on a metric\n\n Args:\n metric: Metric to use for ranking (uses combined_score or average if None)\n\n Returns:\n Best program or None if database is empty\n \"\"\"\n if not self.programs:\n return None\n\n # If no specific metric and we have a tracked best program, return it\n if metric is None and self.best_program_id and self.best_program_id in self.programs:\n logger.debug(f\"Using tracked best program: {self.best_program_id}\")\n return self.programs[self.best_program_id]\n\n if metric:\n # Sort by specific metric\n sorted_programs = sorted(\n [p for p in self.programs.values() if metric in p.metrics],\n key=lambda p: p.metrics[metric],\n reverse=True,\n )\n if sorted_programs:\n logger.debug(f\"Found best program by metric '{metric}': {sorted_programs[0].id}\")\n elif self.programs and all(\"combined_score\" in p.metrics for p in self.programs.values()):\n # Sort by combined_score if it exists (preferred method)\n sorted_programs = sorted(\n self.programs.values(), key=lambda p: p.metrics[\"combined_score\"], reverse=True\n )\n if sorted_programs:\n logger.debug(f\"Found best program by combined_score: {sorted_programs[0].id}\")\n else:\n # Sort by average of all numeric metrics as fallback\n sorted_programs = sorted(\n self.programs.values(),\n key=lambda p: safe_numeric_average(p.metrics),\n reverse=True,\n )\n if sorted_programs:\n logger.debug(f\"Found best program by average metrics: {sorted_programs[0].id}\")\n\n # Update the best program tracking if we found a better program\n if sorted_programs and (\n self.best_program_id is None or sorted_programs[0].id != self.best_program_id\n ):\n old_id = self.best_program_id\n self.best_program_id = sorted_programs[0].id\n logger.info(f\"Updated best program tracking from {old_id} to {self.best_program_id}\")\n\n # Also log the scores to help understand the update\n if (\n old_id\n and old_id in self.programs\n and \"combined_score\" in self.programs[old_id].metrics\n and \"combined_score\" in self.programs[self.best_program_id].metrics\n ):\n old_score = self.programs[old_id].metrics[\"combined_score\"]\n new_score = self.programs[self.best_program_id].metrics[\"combined_score\"]\n logger.info(\n f\"Score change: {old_score:.4f} → {new_score:.4f} ({new_score-old_score:+.4f})\"\n )\n\n return sorted_programs[0] if sorted_programs else None\n\n def get_top_programs(self, n: int = 10, metric: Optional[str] = None) -> List[Program]:\n \"\"\"\n Get the top N programs based on a metric\n\n Args:\n n: Number of programs to return\n metric: Metric to use for ranking (uses average if None)\n\n Returns:\n List of top programs\n \"\"\"\n if not self.programs:\n return []\n\n if metric:\n # Sort by specific metric\n sorted_programs = sorted(\n [p for p in self.programs.values() if metric in p.metrics],\n key=lambda p: p.metrics[metric],\n reverse=True,\n )\n else:\n # Sort by average of all numeric metrics\n sorted_programs = sorted(\n self.programs.values(),\n key=lambda p: safe_numeric_average(p.metrics),\n reverse=True,\n )\n\n return sorted_programs[:n]\n\n def save(self, path: Optional[str] = None, iteration: int = 0) -> None:\n \"\"\"\n Save the database to disk\n\n Args:\n path: Path to save to (uses config.db_path if None)\n iteration: Current iteration number\n \"\"\"\n save_path = path or self.config.db_path\n if not save_path:\n logger.warning(\"No database path specified, skipping save\")\n return\n\n # Create directory if it doesn't exist\n os.makedirs(save_path, exist_ok=True)\n\n # Save each program\n for program in self.programs.values():\n prompts = None\n if (\n self.config.log_prompts\n and self.prompts_by_program\n and program.id in self.prompts_by_program\n ):\n prompts = self.prompts_by_program[program.id]\n self._save_program(program, save_path, prompts=prompts)\n\n # Save metadata\n metadata = {\n \"feature_map\": self.feature_map,\n \"islands\": [list(island) for island in self.islands],\n \"archive\": list(self.archive),\n \"best_program_id\": self.best_program_id,\n \"last_iteration\": iteration or self.last_iteration,\n \"current_island\": self.current_island,\n \"island_generations\": self.island_generations,\n \"last_migration_generation\": self.last_migration_generation,\n }\n\n with open(os.path.join(save_path, \"metadata.json\"), \"w\") as f:\n json.dump(metadata, f)\n\n logger.info(f\"Saved database with {len(self.programs)} programs to {save_path}\")\n\n def load(self, path: str) -> None:\n \"\"\"\n Load the database from disk\n\n Args:\n path: Path to load from\n \"\"\"\n if not os.path.exists(path):\n logger.warning(f\"Database path {path} does not exist, skipping load\")\n return\n\n # Load metadata first\n metadata_path = os.path.join(path, \"metadata.json\")\n saved_islands = []\n if os.path.exists(metadata_path):\n with open(metadata_path, \"r\") as f:\n metadata = json.load(f)\n\n self.feature_map = metadata.get(\"feature_map\", {})\n saved_islands = metadata.get(\"islands\", [])\n self.archive = set(metadata.get(\"archive\", []))\n self.best_program_id = metadata.get(\"best_program_id\")\n self.last_iteration = metadata.get(\"last_iteration\", 0)\n self.current_island = metadata.get(\"current_island\", 0)\n self.island_generations = metadata.get(\"island_generations\", [0] * len(saved_islands))\n self.last_migration_generation = metadata.get(\"last_migration_generation\", 0)\n\n logger.info(f\"Loaded database metadata with last_iteration={self.last_iteration}\")\n\n # Load programs\n programs_dir = os.path.join(path, \"programs\")\n if os.path.exists(programs_dir):\n for program_file in os.listdir(programs_dir):\n if program_file.endswith(\".json\"):\n program_path = os.path.join(programs_dir, program_file)\n try:\n with open(program_path, \"r\") as f:\n program_data = json.load(f)\n\n program = Program.from_dict(program_data)\n self.programs[program.id] = program\n except Exception as e:\n logger.warning(f\"Error loading program {program_file}: {str(e)}\")\n\n # Reconstruct island assignments from metadata\n self._reconstruct_islands(saved_islands)\n\n # Ensure island_generations list has correct length\n if len(self.island_generations) != len(self.islands):\n self.island_generations = [0] * len(self.islands)\n\n logger.info(f\"Loaded database with {len(self.programs)} programs from {path}\")\n\n # Log the reconstructed island status\n self.log_island_status()\n\n def _reconstruct_islands(self, saved_islands: List[List[str]]) -> None:\n \"\"\"\n Reconstruct island assignments from saved metadata\n\n Args:\n saved_islands: List of island program ID lists from metadata\n \"\"\"\n # Initialize empty islands\n num_islands = max(len(saved_islands), self.config.num_islands)\n self.islands = [set() for _ in range(num_islands)]\n\n missing_programs = []\n restored_programs = 0\n\n # Restore island assignments\n for island_idx, program_ids in enumerate(saved_islands):\n if island_idx >= len(self.islands):\n continue\n\n for program_id in program_ids:\n if program_id in self.programs:\n # Program exists, add to island\n self.islands[island_idx].add(program_id)\n # Set island metadata on the program\n self.programs[program_id].metadata[\"island\"] = island_idx\n restored_programs += 1\n else:\n # Program missing, track it\n missing_programs.append((island_idx, program_id))\n\n # Clean up archive - remove missing programs\n original_archive_size = len(self.archive)\n self.archive = {pid for pid in self.archive if pid in self.programs}\n\n # Clean up feature_map - remove missing programs\n feature_keys_to_remove = []\n for key, program_id in self.feature_map.items():\n if program_id not in self.programs:\n feature_keys_to_remove.append(key)\n for key in feature_keys_to_remove:\n del self.feature_map[key]\n\n # Check best program\n if self.best_program_id and self.best_program_id not in self.programs:\n logger.warning(f\"Best program {self.best_program_id} not found, will recalculate\")\n self.best_program_id = None\n\n # Log reconstruction results\n if missing_programs:\n logger.warning(\n f\"Found {len(missing_programs)} missing programs during island reconstruction:\"\n )\n for island_idx, program_id in missing_programs[:5]: # Show first 5\n logger.warning(f\" Island {island_idx}: {program_id}\")\n if len(missing_programs) > 5:\n logger.warning(f\" ... and {len(missing_programs) - 5} more\")\n\n if original_archive_size > len(self.archive):\n logger.info(\n f\"Removed {original_archive_size - len(self.archive)} missing programs from archive\"\n )\n\n if feature_keys_to_remove:\n logger.info(f\"Removed {len(feature_keys_to_remove)} missing programs from feature map\")\n\n logger.info(f\"Reconstructed islands: restored {restored_programs} programs to islands\")\n\n # If we have programs but no island assignments, distribute them\n if self.programs and sum(len(island) for island in self.islands) == 0:\n logger.info(\"No island assignments found, distributing programs across islands\")\n self._distribute_programs_to_islands()\n\n def _distribute_programs_to_islands(self) -> None:\n \"\"\"\n Distribute loaded programs across islands when no island metadata exists\n \"\"\"\n program_ids = list(self.programs.keys())\n\n # Distribute programs round-robin across islands\n for i, program_id in enumerate(program_ids):\n island_idx = i % len(self.islands)\n self.islands[island_idx].add(program_id)\n self.programs[program_id].metadata[\"island\"] = island_idx\n\n logger.info(f\"Distributed {len(program_ids)} programs across {len(self.islands)} islands\")\n\n def _save_program(\n self,\n program: Program,\n base_path: Optional[str] = None,\n prompts: Optional[Dict[str, Dict[str, str]]] = None,\n ) -> None:\n \"\"\"\n Save a program to disk\n\n Args:\n program: Program to save\n base_path: Base path to save to (uses config.db_path if None)\n prompts: Optional prompts to save with the program, in the format {template_key: { 'system': str, 'user': str }}\n \"\"\"\n save_path = base_path or self.config.db_path\n if not save_path:\n return\n\n # Create programs directory if it doesn't exist\n programs_dir = os.path.join(save_path, \"programs\")\n os.makedirs(programs_dir, exist_ok=True)\n\n # Save program\n program_dict = program.to_dict()\n if prompts:\n program_dict[\"prompts\"] = prompts\n program_path = os.path.join(programs_dir, f\"{program.id}.json\")\n\n with open(program_path, \"w\") as f:\n json.dump(program_dict, f)\n\n def _calculate_feature_coords(self, program: Program) -> List[int]:\n \"\"\"\n Calculate feature coordinates for the MAP-Elites grid\n\n Args:\n program: Program to calculate features for\n\n Returns:\n List of feature coordinates\n \"\"\"\n coords = []\n\n for dim in self.config.feature_dimensions:\n if dim == \"complexity\":\n # Use code length as complexity measure\n complexity = len(program.code)\n bin_idx = min(int(complexity / 1000 * self.feature_bins), self.feature_bins - 1)\n coords.append(bin_idx)\n elif dim == \"diversity\":\n # Use average edit distance to other programs\n if len(self.programs) < 5:\n bin_idx = 0\n else:\n sample_programs = random.sample(\n list(self.programs.values()), min(5, len(self.programs))\n )\n avg_distance = sum(\n calculate_edit_distance(program.code, other.code)\n for other in sample_programs\n ) / len(sample_programs)\n bin_idx = min(\n int(avg_distance / 1000 * self.feature_bins), self.feature_bins - 1\n )\n coords.append(bin_idx)\n elif dim == \"score\":\n # Use average of numeric metrics\n if not program.metrics:\n bin_idx = 0\n else:\n avg_score = safe_numeric_average(program.metrics)\n bin_idx = min(int(avg_score * self.feature_bins), self.feature_bins - 1)\n coords.append(bin_idx)\n elif dim in program.metrics:\n # Use specific metric\n score = program.metrics[dim]\n bin_idx = min(int(score * self.feature_bins), self.feature_bins - 1)\n coords.append(bin_idx)\n else:\n # Default to middle bin if feature not found\n coords.append(self.feature_bins // 2)\n\n return coords\n\n def _feature_coords_to_key(self, coords: List[int]) -> str:\n \"\"\"\n Convert feature coordinates to a string key\n\n Args:\n coords: Feature coordinates\n\n Returns:\n String key\n \"\"\"\n return \"-\".join(str(c) for c in coords)\n\n def _is_better(self, program1: Program, program2: Program) -> bool:\n \"\"\"\n Determine if program1 is better than program2\n\n Args:\n program1: First program\n program2: Second program\n\n Returns:\n True if program1 is better than program2\n \"\"\"\n # If no metrics, use newest\n if not program1.metrics and not program2.metrics:\n return program1.timestamp > program2.timestamp\n\n # If only one has metrics, it's better\n if program1.metrics and not program2.metrics:\n return True\n if not program1.metrics and program2.metrics:\n return False\n\n # Check for combined_score first (this is the preferred metric)\n if \"combined_score\" in program1.metrics and \"combined_score\" in program2.metrics:\n return program1.metrics[\"combined_score\"] > program2.metrics[\"combined_score\"]\n\n # Fallback to average of all numeric metrics\n avg1 = safe_numeric_average(program1.metrics)\n avg2 = safe_numeric_average(program2.metrics)\n\n return avg1 > avg2\n\n def _update_archive(self, program: Program) -> None:\n \"\"\"\n Update the archive of elite programs\n\n Args:\n program: Program to consider for archive\n \"\"\"\n # If archive not full, add program\n if len(self.archive) < self.config.archive_size:\n self.archive.add(program.id)\n return\n\n # Clean up stale references and get valid archive programs\n valid_archive_programs = []\n stale_ids = []\n\n for pid in self.archive:\n if pid in self.programs:\n valid_archive_programs.append(self.programs[pid])\n else:\n stale_ids.append(pid)\n\n # Remove stale references from archive\n for stale_id in stale_ids:\n self.archive.discard(stale_id)\n logger.debug(f\"Removing stale program {stale_id} from archive\")\n\n # If archive is now not full after cleanup, just add the new program\n if len(self.archive) < self.config.archive_size:\n self.archive.add(program.id)\n return\n\n # Find worst program among valid programs\n if valid_archive_programs:\n worst_program = min(\n valid_archive_programs, key=lambda p: safe_numeric_average(p.metrics)\n )\n\n # Replace if new program is better\n if self._is_better(program, worst_program):\n self.archive.remove(worst_program.id)\n self.archive.add(program.id)\n else:\n # No valid programs in archive, just add the new one\n self.archive.add(program.id)\n\n def _update_best_program(self, program: Program) -> None:\n \"\"\"\n Update the absolute best program tracking\n\n Args:\n program: Program to consider as the new best\n \"\"\"\n # If we don't have a best program yet, this becomes the best\n if self.best_program_id is None:\n self.best_program_id = program.id\n logger.debug(f\"Set initial best program to {program.id}\")\n return\n\n # Compare with current best program\n current_best = self.programs[self.best_program_id]\n\n # Update if the new program is better\n if self._is_better(program, current_best):\n old_id = self.best_program_id\n self.best_program_id = program.id\n\n # Log the change\n if \"combined_score\" in program.metrics and \"combined_score\" in current_best.metrics:\n old_score = current_best.metrics[\"combined_score\"]\n new_score = program.metrics[\"combined_score\"]\n score_diff = new_score - old_score\n logger.info(\n f\"New best program {program.id} replaces {old_id} (combined_score: {old_score:.4f} → {new_score:.4f}, +{score_diff:.4f})\"\n )\n else:\n logger.info(f\"New best program {program.id} replaces {old_id}\")\n\n def _sample_parent(self) -> Program:\n \"\"\"\n Sample a parent program from the current island for the next evolution step\n\n Returns:\n Parent program from current island\n \"\"\"\n # Use exploration_ratio and exploitation_ratio to decide sampling strategy\n rand_val = random.random()\n\n if rand_val < self.config.exploration_ratio:\n # EXPLORATION: Sample from current island (diverse sampling)\n return self._sample_exploration_parent()\n elif rand_val < self.config.exploration_ratio + self.config.exploitation_ratio:\n # EXPLOITATION: Sample from archive (elite programs)\n return self._sample_exploitation_parent()\n else:\n # RANDOM: Sample from any program (remaining probability)\n return self._sample_random_parent()\n\n def _sample_exploration_parent(self) -> Program:\n \"\"\"\n Sample a parent for exploration (from current island)\n \"\"\"\n current_island_programs = self.islands[self.current_island]\n\n if not current_island_programs:\n # If current island is empty, initialize with best program or random program\n if self.best_program_id and self.best_program_id in self.programs:\n # Clone best program to current island\n best_program = self.programs[self.best_program_id]\n self.islands[self.current_island].add(self.best_program_id)\n best_program.metadata[\"island\"] = self.current_island\n logger.debug(f\"Initialized empty island {self.current_island} with best program\")\n return best_program\n else:\n # Use any available program\n return next(iter(self.programs.values()))\n\n # Clean up stale references and sample from current island\n valid_programs = [pid for pid in current_island_programs if pid in self.programs]\n\n # Remove stale program IDs from island\n if len(valid_programs) < len(current_island_programs):\n stale_ids = current_island_programs - set(valid_programs)\n logger.debug(\n f\"Removing {len(stale_ids)} stale program IDs from island {self.current_island}\"\n )\n for stale_id in stale_ids:\n self.islands[self.current_island].discard(stale_id)\n\n # If no valid programs after cleanup, reinitialize island\n if not valid_programs:\n logger.warning(\n f\"Island {self.current_island} has no valid programs after cleanup, reinitializing\"\n )\n if self.best_program_id and self.best_program_id in self.programs:\n best_program = self.programs[self.best_program_id]\n self.islands[self.current_island].add(self.best_program_id)\n best_program.metadata[\"island\"] = self.current_island\n return best_program\n else:\n return next(iter(self.programs.values()))\n\n # Sample from valid programs\n parent_id = random.choice(valid_programs)\n return self.programs[parent_id]\n\n def _sample_exploitation_parent(self) -> Program:\n \"\"\"\n Sample a parent for exploitation (from archive/elite programs)\n \"\"\"\n if not self.archive:\n # Fallback to exploration if no archive\n return self._sample_exploration_parent()\n\n # Clean up stale references in archive\n valid_archive = [pid for pid in self.archive if pid in self.programs]\n\n # Remove stale program IDs from archive\n if len(valid_archive) < len(self.archive):\n stale_ids = self.archive - set(valid_archive)\n logger.debug(f\"Removing {len(stale_ids)} stale program IDs from archive\")\n for stale_id in stale_ids:\n self.archive.discard(stale_id)\n\n # If no valid archive programs, fallback to exploration\n if not valid_archive:\n logger.warning(\n \"Archive has no valid programs after cleanup, falling back to exploration\"\n )\n return self._sample_exploration_parent()\n\n # Prefer programs from current island in archive\n archive_programs_in_island = [\n pid\n for pid in valid_archive\n if self.programs[pid].metadata.get(\"island\") == self.current_island\n ]\n\n if archive_programs_in_island:\n parent_id = random.choice(archive_programs_in_island)\n return self.programs[parent_id]\n else:\n # Fall back to any valid archive program if current island has none\n parent_id = random.choice(valid_archive)\n return self.programs[parent_id]\n\n def _sample_random_parent(self) -> Program:\n \"\"\"\n Sample a completely random parent from all programs\n \"\"\"\n if not self.programs:\n raise ValueError(\"No programs available for sampling\")\n\n # Sample randomly from all programs\n program_id = random.choice(list(self.programs.keys()))\n return self.programs[program_id]\n\n def _sample_inspirations(self, parent: Program, n: int = 5) -> List[Program]:\n \"\"\"\n Sample inspiration programs for the next evolution step\n\n Args:\n parent: Parent program\n n: Number of inspirations to sample\n\n Returns:\n List of inspiration programs\n \"\"\"\n inspirations = []\n\n # Always include the absolute best program if available and different from parent\n if (\n self.best_program_id is not None\n and self.best_program_id != parent.id\n and self.best_program_id in self.programs\n ):\n best_program = self.programs[self.best_program_id]\n inspirations.append(best_program)\n logger.debug(f\"Including best program {self.best_program_id} in inspirations\")\n elif self.best_program_id is not None and self.best_program_id not in self.programs:\n # Clean up stale best program reference\n logger.warning(\n f\"Best program {self.best_program_id} no longer exists, clearing reference\"\n )\n self.best_program_id = None\n\n # Add top programs as inspirations\n top_n = max(1, int(n * self.config.elite_selection_ratio))\n top_programs = self.get_top_programs(n=top_n)\n for program in top_programs:\n if program.id not in [p.id for p in inspirations] and program.id != parent.id:\n inspirations.append(program)\n\n # Add diverse programs using config.num_diverse_programs\n if len(self.programs) > n and len(inspirations) < n:\n # Calculate how many diverse programs to add (up to remaining slots)\n remaining_slots = n - len(inspirations)\n\n # Sample from different feature cells for diversity\n feature_coords = self._calculate_feature_coords(parent)\n\n # Get programs from nearby feature cells\n nearby_programs = []\n for _ in range(remaining_slots):\n # Perturb coordinates\n perturbed_coords = [\n max(0, min(self.feature_bins - 1, c + random.randint(-1, 1)))\n for c in feature_coords\n ]\n\n # Try to get program from this cell\n cell_key = self._feature_coords_to_key(perturbed_coords)\n if cell_key in self.feature_map:\n program_id = self.feature_map[cell_key]\n # Check if program still exists before adding\n if (\n program_id != parent.id\n and program_id not in [p.id for p in inspirations]\n and program_id in self.programs\n ):\n nearby_programs.append(self.programs[program_id])\n elif program_id not in self.programs:\n # Clean up stale reference in feature_map\n logger.debug(f\"Removing stale program {program_id} from feature_map\")\n del self.feature_map[cell_key]\n\n # If we need more, add random programs\n if len(inspirations) + len(nearby_programs) < n:\n remaining = n - len(inspirations) - len(nearby_programs)\n all_ids = set(self.programs.keys())\n excluded_ids = (\n {parent.id}\n .union(p.id for p in inspirations)\n .union(p.id for p in nearby_programs)\n )\n available_ids = list(all_ids - excluded_ids)\n\n if available_ids:\n random_ids = random.sample(available_ids, min(remaining, len(available_ids)))\n random_programs = [self.programs[pid] for pid in random_ids]\n nearby_programs.extend(random_programs)\n\n inspirations.extend(nearby_programs)\n\n return inspirations[:n]\n\n def _enforce_population_limit(self) -> None:\n \"\"\"\n Enforce the population size limit by removing worst programs if needed\n \"\"\"\n if len(self.programs) <= self.config.population_size:\n return\n\n # Calculate how many programs to remove\n num_to_remove = len(self.programs) - self.config.population_size\n\n logger.info(\n f\"Population size ({len(self.programs)}) exceeds limit ({self.config.population_size}), removing {num_to_remove} programs\"\n )\n\n # Get programs sorted by fitness (worst first)\n all_programs = list(self.programs.values())\n\n # Sort by average metric (worst first)\n sorted_programs = sorted(\n all_programs,\n key=lambda p: safe_numeric_average(p.metrics),\n )\n\n # Remove worst programs, but never remove the best program\n programs_to_remove = []\n for program in sorted_programs:\n if len(programs_to_remove) >= num_to_remove:\n break\n # Don't remove the best program\n if program.id != self.best_program_id:\n programs_to_remove.append(program)\n\n # If we still need to remove more and only have the best program protected,\n # remove from the remaining programs anyway (but keep the absolute best)\n if len(programs_to_remove) < num_to_remove:\n remaining_programs = [\n p\n for p in sorted_programs\n if p not in programs_to_remove and p.id != self.best_program_id\n ]\n additional_removals = remaining_programs[: num_to_remove - len(programs_to_remove)]\n programs_to_remove.extend(additional_removals)\n\n # Remove the selected programs\n for program in programs_to_remove:\n program_id = program.id\n\n # Remove from main programs dict\n if program_id in self.programs:\n del self.programs[program_id]\n\n # Remove from feature map\n keys_to_remove = []\n for key, pid in self.feature_map.items():\n if pid == program_id:\n keys_to_remove.append(key)\n for key in keys_to_remove:\n del self.feature_map[key]\n\n # Remove from islands\n for island in self.islands:\n island.discard(program_id)\n\n # Remove from archive\n self.archive.discard(program_id)\n\n logger.debug(f\"Removed program {program_id} due to population limit\")\n\n logger.info(f\"Population size after cleanup: {len(self.programs)}\")\n\n # Island management methods\n def set_current_island(self, island_idx: int) -> None:\n \"\"\"Set which island is currently being evolved\"\"\"\n self.current_island = island_idx % len(self.islands)\n logger.debug(f\"Switched to evolving island {self.current_island}\")\n\n def next_island(self) -> int:\n \"\"\"Move to the next island in round-robin fashion\"\"\"\n self.current_island = (self.current_island + 1) % len(self.islands)\n logger.debug(f\"Advanced to island {self.current_island}\")\n return self.current_island\n\n def increment_island_generation(self, island_idx: Optional[int] = None) -> None:\n \"\"\"Increment generation counter for an island\"\"\"\n idx = island_idx if island_idx is not None else self.current_island\n self.island_generations[idx] += 1\n logger.debug(f\"Island {idx} generation incremented to {self.island_generations[idx]}\")\n\n def should_migrate(self) -> bool:\n \"\"\"Check if migration should occur based on generation counters\"\"\"\n max_generation = max(self.island_generations)\n return (max_generation - self.last_migration_generation) >= self.migration_interval\n\n def migrate_programs(self) -> None:\n \"\"\"\n Perform migration between islands\n\n This should be called periodically to share good solutions between islands\n \"\"\"\n if len(self.islands) < 2:\n return\n\n logger.info(\"Performing migration between islands\")\n\n for i, island in enumerate(self.islands):\n if len(island) == 0:\n continue\n\n # Select top programs from this island for migration\n island_programs = [self.programs[pid] for pid in island if pid in self.programs]\n if not island_programs:\n continue\n\n # Sort by fitness (using combined_score or average metrics)\n island_programs.sort(\n key=lambda p: p.metrics.get(\"combined_score\", safe_numeric_average(p.metrics)),\n reverse=True,\n )\n\n # Select top programs for migration\n num_to_migrate = max(1, int(len(island_programs) * self.migration_rate))\n migrants = island_programs[:num_to_migrate]\n\n # Migrate to adjacent islands (ring topology)\n target_islands = [(i + 1) % len(self.islands), (i - 1) % len(self.islands)]\n\n for migrant in migrants:\n for target_island in target_islands:\n # Create a copy for migration (to avoid removing from source)\n migrant_copy = Program(\n id=f\"{migrant.id}_migrant_{target_island}\",\n code=migrant.code,\n language=migrant.language,\n parent_id=migrant.id,\n generation=migrant.generation,\n metrics=migrant.metrics.copy(),\n metadata={**migrant.metadata, \"island\": target_island, \"migrant\": True},\n )\n\n # Add to target island\n self.islands[target_island].add(migrant_copy.id)\n self.programs[migrant_copy.id] = migrant_copy\n\n logger.debug(\n f\"Migrated program {migrant.id} from island {i} to island {target_island}\"\n )\n\n # Update last migration generation\n self.last_migration_generation = max(self.island_generations)\n logger.info(f\"Migration completed at generation {self.last_migration_generation}\")\n\n def get_island_stats(self) -> List[dict]:\n \"\"\"Get statistics for each island\"\"\"\n stats = []\n\n for i, island in enumerate(self.islands):\n island_programs = [self.programs[pid] for pid in island if pid in self.programs]\n\n if island_programs:\n scores = [\n p.metrics.get(\"combined_score\", safe_numeric_average(p.metrics))\n for p in island_programs\n ]\n\n best_score = max(scores) if scores else 0.0\n avg_score = sum(scores) / len(scores) if scores else 0.0\n diversity = self._calculate_island_diversity(island_programs)\n else:\n best_score = avg_score = diversity = 0.0\n\n stats.append(\n {\n \"island\": i,\n \"population_size\": len(island_programs),\n \"best_score\": best_score,\n \"average_score\": avg_score,\n \"diversity\": diversity,\n \"generation\": self.island_generations[i],\n \"is_current\": i == self.current_island,\n }\n )\n\n return stats\n\n def _calculate_island_diversity(self, programs: List[Program]) -> float:\n \"\"\"Calculate diversity within an island (deterministic version)\"\"\"\n if len(programs) < 2:\n return 0.0\n\n total_diversity = 0\n comparisons = 0\n\n # Use deterministic sampling instead of random.sample() to ensure consistent results\n sample_size = min(5, len(programs)) # Reduced from 10 to 5\n\n # Sort programs by ID for deterministic ordering\n sorted_programs = sorted(programs, key=lambda p: p.id)\n\n # Take first N programs instead of random sampling\n sample_programs = sorted_programs[:sample_size]\n\n # Limit total comparisons for performance\n max_comparisons = 6 # Maximum comparisons to prevent long delays\n\n for i, prog1 in enumerate(sample_programs):\n for prog2 in sample_programs[i + 1 :]:\n if comparisons >= max_comparisons:\n break\n\n # Use fast approximation instead of expensive edit distance\n diversity = self._fast_code_diversity(prog1.code, prog2.code)\n total_diversity += diversity\n comparisons += 1\n\n if comparisons >= max_comparisons:\n break\n\n return total_diversity / max(1, comparisons)\n\n def _fast_code_diversity(self, code1: str, code2: str) -> float:\n \"\"\"\n Fast approximation of code diversity using simple metrics\n\n Returns diversity score (higher = more diverse)\n \"\"\"\n if code1 == code2:\n return 0.0\n\n # Length difference (scaled to reasonable range)\n len1, len2 = len(code1), len(code2)\n length_diff = abs(len1 - len2)\n\n # Line count difference\n lines1 = code1.count(\"\\n\")\n lines2 = code2.count(\"\\n\")\n line_diff = abs(lines1 - lines2)\n\n # Simple character set difference\n chars1 = set(code1)\n chars2 = set(code2)\n char_diff = len(chars1.symmetric_difference(chars2))\n\n # Combine metrics (scaled to match original edit distance range)\n diversity = length_diff * 0.1 + line_diff * 10 + char_diff * 0.5\n\n return diversity\n\n def log_island_status(self) -> None:\n \"\"\"Log current status of all islands\"\"\"\n stats = self.get_island_stats()\n logger.info(\"Island Status:\")\n for stat in stats:\n current_marker = \" *\" if stat[\"is_current\"] else \" \"\n logger.info(\n f\"{current_marker} Island {stat['island']}: {stat['population_size']} programs, \"\n f\"best={stat['best_score']:.4f}, avg={stat['average_score']:.4f}, \"\n f\"diversity={stat['diversity']:.2f}, gen={stat['generation']}\"\n )\n\n # Artifact storage and retrieval methods\n\n def store_artifacts(self, program_id: str, artifacts: Dict[str, Union[str, bytes]]) -> None:\n \"\"\"\n Store artifacts for a program\n\n Args:\n program_id: ID of the program\n artifacts: Dictionary of artifact name to content\n \"\"\"\n if not artifacts:\n return\n\n program = self.get(program_id)\n if not program:\n logger.warning(f\"Cannot store artifacts: program {program_id} not found\")\n return\n\n # Check if artifacts are enabled\n artifacts_enabled = os.environ.get(\"ENABLE_ARTIFACTS\", \"true\").lower() == \"true\"\n if not artifacts_enabled:\n logger.debug(\"Artifacts disabled, skipping storage\")\n return\n\n # Split artifacts by size\n small_artifacts = {}\n large_artifacts = {}\n size_threshold = getattr(self.config, \"artifact_size_threshold\", 32 * 1024) # 32KB default\n\n for key, value in artifacts.items():\n size = self._get_artifact_size(value)\n if size <= size_threshold:\n small_artifacts[key] = value\n else:\n large_artifacts[key] = value\n\n # Store small artifacts as JSON\n if small_artifacts:\n program.artifacts_json = json.dumps(small_artifacts, default=self._artifact_serializer)\n logger.debug(f\"Stored {len(small_artifacts)} small artifacts for program {program_id}\")\n\n # Store large artifacts to disk\n if large_artifacts:\n artifact_dir = self._create_artifact_dir(program_id)\n program.artifact_dir = artifact_dir\n for key, value in large_artifacts.items():\n self._write_artifact_file(artifact_dir, key, value)\n logger.debug(f\"Stored {len(large_artifacts)} large artifacts for program {program_id}\")\n\n def get_artifacts(self, program_id: str) -> Dict[str, Union[str, bytes]]:\n \"\"\"\n Retrieve all artifacts for a program\n\n Args:\n program_id: ID of the program\n\n Returns:\n Dictionary of artifact name to content\n \"\"\"\n program = self.get(program_id)\n if not program:\n return {}\n\n artifacts = {}\n\n # Load small artifacts from JSON\n if program.artifacts_json:\n try:\n small_artifacts = json.loads(program.artifacts_json)\n artifacts.update(small_artifacts)\n except json.JSONDecodeError as e:\n logger.warning(f\"Failed to decode artifacts JSON for program {program_id}: {e}\")\n\n # Load large artifacts from disk\n if program.artifact_dir and os.path.exists(program.artifact_dir):\n disk_artifacts = self._load_artifact_dir(program.artifact_dir)\n artifacts.update(disk_artifacts)\n\n return artifacts\n\n def _get_artifact_size(self, value: Union[str, bytes]) -> int:\n \"\"\"Get size of an artifact value in bytes\"\"\"\n if isinstance(value, str):\n return len(value.encode(\"utf-8\"))\n elif isinstance(value, bytes):\n return len(value)\n else:\n return len(str(value).encode(\"utf-8\"))\n\n def _artifact_serializer(self, obj):\n \"\"\"JSON serializer for artifacts that handles bytes\"\"\"\n if isinstance(obj, bytes):\n return {\"__bytes__\": base64.b64encode(obj).decode(\"utf-8\")}\n raise TypeError(f\"Object of type {type(obj)} is not JSON serializable\")\n\n def _artifact_deserializer(self, dct):\n \"\"\"JSON deserializer for artifacts that handles bytes\"\"\"\n if \"__bytes__\" in dct:\n return base64.b64decode(dct[\"__bytes__\"])\n return dct\n\n def _create_artifact_dir(self, program_id: str) -> str:\n \"\"\"Create artifact directory for a program\"\"\"\n base_path = getattr(self.config, \"artifacts_base_path\", None)\n if not base_path:\n base_path = (\n os.path.join(self.config.db_path or \".\", \"artifacts\")\n if self.config.db_path\n else \"./artifacts\"\n )\n\n artifact_dir = os.path.join(base_path, program_id)\n os.makedirs(artifact_dir, exist_ok=True)\n return artifact_dir\n\n def _write_artifact_file(self, artifact_dir: str, key: str, value: Union[str, bytes]) -> None:\n \"\"\"Write an artifact to a file\"\"\"\n # Sanitize filename\n safe_key = \"\".join(c for c in key if c.isalnum() or c in \"._-\")\n if not safe_key:\n safe_key = \"artifact\"\n\n file_path = os.path.join(artifact_dir, safe_key)\n\n try:\n if isinstance(value, str):\n with open(file_path, \"w\", encoding=\"utf-8\") as f:\n f.write(value)\n elif isinstance(value, bytes):\n with open(file_path, \"wb\") as f:\n f.write(value)\n else:\n # Convert to string and write\n with open(file_path, \"w\", encoding=\"utf-8\") as f:\n f.write(str(value))\n except Exception as e:\n logger.warning(f\"Failed to write artifact {key} to {file_path}: {e}\")\n\n def _load_artifact_dir(self, artifact_dir: str) -> Dict[str, Union[str, bytes]]:\n \"\"\"Load artifacts from a directory\"\"\"\n artifacts = {}\n\n try:\n for filename in os.listdir(artifact_dir):\n file_path = os.path.join(artifact_dir, filename)\n if os.path.isfile(file_path):\n try:\n # Try to read as text first\n with open(file_path, \"r\", encoding=\"utf-8\") as f:\n content = f.read()\n artifacts[filename] = content\n except UnicodeDecodeError:\n # If text fails, read as binary\n with open(file_path, \"rb\") as f:\n content = f.read()\n artifacts[filename] = content\n except Exception as e:\n logger.warning(f\"Failed to read artifact file {file_path}: {e}\")\n except Exception as e:\n logger.warning(f\"Failed to list artifact directory {artifact_dir}: {e}\")\n\n return artifacts\n\n def log_prompt(\n self,\n program_id: str,\n template_key: str,\n prompt: Dict[str, str],\n responses: Optional[List[str]] = None,\n ) -> None:\n \"\"\"\n Log a prompt for a program.\n Only logs if self.config.log_prompts is True.\n\n Args:\n program_id: ID of the program to log the prompt for\n template_key: Key for the prompt template\n prompt: Prompts in the format {template_key: { 'system': str, 'user': str }}.\n responses: Optional list of responses to the prompt, if available.\n \"\"\"\n\n if not self.config.log_prompts:\n return\n\n if responses is None:\n responses = []\n prompt[\"responses\"] = responses\n\n if self.prompts_by_program is None:\n self.prompts_by_program = {}\n\n if program_id not in self.prompts_by_program:\n self.prompts_by_program[program_id] = {}\n self.prompts_by_program[program_id][template_key] = prompt\n"}}, "evolve_agent/evaluator.py": {"7851c26290fb9e1a23b9f41261ea0cacafe99579": {"t": 1785599212.8579469, "n": 25005, "h": "78dbfca2639eab36", "c": "\"\"\"\nEvaluation system for EvolveAgent\n\"\"\"\n\nimport asyncio\nimport importlib.util\nimport json\nimport logging\nimport os\nimport subprocess\nimport sys\nimport tempfile\nimport time\nimport traceback\nimport uuid\nfrom pathlib import Path\nfrom typing import Any, Callable, Dict, List, Optional, Tuple, Union\nimport traceback\n\nfrom evolve_agent.config import EvaluatorConfig\nfrom evolve_agent.database import ProgramDatabase\nfrom evolve_agent.evaluation_result import EvaluationResult\nfrom evolve_agent.database import ProgramDatabase\nfrom evolve_agent.llm.ensemble import LLMEnsemble\nfrom evolve_agent.utils.async_utils import TaskPool, run_in_executor\nfrom evolve_agent.prompt.sampler import PromptSampler\nfrom evolve_agent.utils.format_utils import format_metrics_safe\n\nlogger = logging.getLogger(__name__)\n\n\nclass Evaluator:\n \"\"\"\n Evaluates programs and assigns scores\n\n The evaluator is responsible for executing programs, measuring their performance,\n and assigning scores based on the evaluation criteria.\n \"\"\"\n\n def __init__(\n self,\n config: EvaluatorConfig,\n evaluation_file: str,\n llm_ensemble: Optional[LLMEnsemble] = None,\n prompt_sampler: Optional[PromptSampler] = None,\n database: Optional[ProgramDatabase] = None,\n ):\n self.config = config\n self.evaluation_file = evaluation_file\n self.llm_ensemble = llm_ensemble\n self.prompt_sampler = prompt_sampler\n self.database = database\n\n # Create a task pool for parallel evaluation\n self.task_pool = TaskPool(max_concurrency=config.parallel_evaluations)\n\n # Set up evaluation function if file exists\n self._load_evaluation_function()\n\n # Pending artifacts storage for programs\n self._pending_artifacts: Dict[str, Dict[str, Union[str, bytes]]] = {}\n\n logger.info(f\"Initialized evaluator with {evaluation_file}\")\n\n def _load_evaluation_function(self) -> None:\n \"\"\"Load the evaluation function from the evaluation file\"\"\"\n if not os.path.exists(self.evaluation_file):\n raise ValueError(f\"Evaluation file {self.evaluation_file} not found\")\n\n try:\n # Add the evaluation file's directory to Python path so it can import local modules\n eval_dir = os.path.dirname(os.path.abspath(self.evaluation_file))\n if eval_dir not in sys.path:\n sys.path.insert(0, eval_dir)\n logger.debug(f\"Added {eval_dir} to Python path for local imports\")\n\n spec = importlib.util.spec_from_file_location(\"evaluation_module\", self.evaluation_file)\n if spec is None or spec.loader is None:\n raise ImportError(f\"Failed to load spec from {self.evaluation_file}\")\n\n module = importlib.util.module_from_spec(spec)\n sys.modules[\"evaluation_module\"] = module\n spec.loader.exec_module(module)\n\n if not hasattr(module, \"evaluate\"):\n raise AttributeError(\n f\"Evaluation file {self.evaluation_file} does not contain an 'evaluate' function\"\n )\n\n self.evaluate_function = module.evaluate\n logger.info(f\"Successfully loaded evaluation function from {self.evaluation_file}\")\n except Exception as e:\n logger.error(f\"Error loading evaluation function: {str(e)}\")\n raise\n\n async def evaluate_program(\n self,\n program_code: str,\n program_id: str = \"\",\n ) -> Dict[str, float]:\n \"\"\"\n Evaluate a program and return scores\n\n Args:\n program_code: Code to evaluate\n program_id: Optional ID for logging\n\n Returns:\n Dictionary of metric name to score\n \"\"\"\n start_time = time.time()\n program_id_str = f\" {program_id}\" if program_id else \"\"\n\n # Check if artifacts are enabled\n artifacts_enabled = os.environ.get(\"ENABLE_ARTIFACTS\", \"true\").lower() == \"true\"\n\n # Retry logic for evaluation\n last_exception = None\n for attempt in range(self.config.max_retries + 1):\n # Create a temporary file for the program\n with tempfile.NamedTemporaryFile(suffix=\".py\", delete=False) as temp_file:\n temp_file.write(program_code.encode(\"utf-8\"))\n temp_file_path = temp_file.name\n\n try:\n # Run evaluation\n if self.config.cascade_evaluation:\n # Run cascade evaluation\n result = await self._cascade_evaluate(temp_file_path)\n else:\n # Run direct evaluation\n result = await self._direct_evaluate(temp_file_path)\n\n # Process the result based on type\n eval_result = self._process_evaluation_result(result)\n\n # Check if this was a timeout and capture artifacts if enabled\n if artifacts_enabled and program_id and eval_result.metrics.get(\"timeout\") is True:\n if program_id not in self._pending_artifacts:\n self._pending_artifacts[program_id] = {}\n\n self._pending_artifacts[program_id].update(\n {\n \"timeout\": True,\n \"timeout_duration\": self.config.timeout,\n \"failure_stage\": \"evaluation\",\n \"error_type\": \"timeout\",\n }\n )\n\n # Add LLM feedback if configured\n llm_eval_result = None\n if self.config.use_llm_feedback and self.llm_ensemble:\n llm_result = await self._llm_evaluate(program_code, program_id=program_id)\n llm_eval_result = self._process_evaluation_result(llm_result)\n\n # Combine metrics\n for name, value in llm_result.metrics.items():\n eval_result.metrics[f\"llm_{name}\"] = value * self.config.llm_feedback_weight\n\n # Store artifacts if enabled and present\n if (\n artifacts_enabled\n and (\n eval_result.has_artifacts()\n or (llm_eval_result and llm_eval_result.has_artifacts())\n )\n and program_id\n ):\n if program_id not in self._pending_artifacts:\n self._pending_artifacts[program_id] = {}\n\n # Merge eval_result artifacts with llm artifacts if they exist\n if eval_result.has_artifacts():\n self._pending_artifacts[program_id].update(eval_result.artifacts)\n logger.debug(\n f\"Program{program_id_str} returned artifacts: \"\n f\"{eval_result.artifacts}\"\n )\n\n if llm_eval_result and llm_eval_result.has_artifacts():\n self._pending_artifacts[program_id].update(llm_eval_result.artifacts)\n logger.debug(\n f\"Program{program_id_str} returned LLM artifacts: \"\n f\"{llm_eval_result.artifacts}\"\n )\n\n elapsed = time.time() - start_time\n logger.info(\n f\"Evaluated program{program_id_str} in {elapsed:.2f}s: \"\n f\"{format_metrics_safe(eval_result.metrics)}\"\n )\n\n # Return just metrics for backward compatibility\n return eval_result.metrics\n\n except asyncio.TimeoutError:\n # Handle timeout specially - don't retry, just return timeout result\n logger.warning(f\"Evaluation timed out after {self.config.timeout}s\")\n\n # Capture timeout artifacts if enabled\n if artifacts_enabled and program_id:\n self._pending_artifacts[program_id] = {\n \"timeout\": True,\n \"timeout_duration\": self.config.timeout,\n \"failure_stage\": \"evaluation\",\n \"error_type\": \"timeout\",\n }\n\n return {\"error\": -1.0, \"timeout\": True}\n\n except Exception as e:\n last_exception = e\n logger.warning(\n f\"Evaluation attempt {attempt + 1}/{self.config.max_retries + 1} failed for program{program_id_str}: {str(e)}\"\n )\n traceback.print_exc()\n\n # Capture failure artifacts if enabled\n if artifacts_enabled and program_id:\n self._pending_artifacts[program_id] = {\n \"stderr\": str(e),\n \"traceback\": traceback.format_exc(),\n \"failure_stage\": \"evaluation\",\n \"attempt\": attempt + 1,\n }\n\n # If this is not the last attempt, wait a bit before retrying\n if attempt < self.config.max_retries:\n await asyncio.sleep(1.0) # Wait 1 second before retry\n\n finally:\n # Clean up temporary file\n if os.path.exists(temp_file_path):\n os.unlink(temp_file_path)\n\n # All retries failed\n logger.error(\n f\"All evaluation attempts failed for program{program_id_str}. Last error: {str(last_exception)}\"\n )\n return {\"error\": -1.0}\n\n def _process_evaluation_result(self, result: Any) -> EvaluationResult:\n \"\"\"\n Process evaluation result to handle both dict and EvaluationResult returns\n\n Args:\n result: Raw result from evaluation function\n\n Returns:\n EvaluationResult instance\n \"\"\"\n if isinstance(result, dict):\n # Backward compatibility - wrap dict in EvaluationResult\n return EvaluationResult.from_dict(result)\n elif isinstance(result, EvaluationResult):\n # New format - use directly\n return result\n else:\n # Error case - return error metrics\n logger.warning(f\"Unexpected evaluation result type: {type(result)}\")\n return EvaluationResult(metrics={\"error\": -1.0})\n\n def get_pending_artifacts(self, program_id: str) -> Optional[Dict[str, Union[str, bytes]]]:\n \"\"\"\n Get and clear pending artifacts for a program\n\n Args:\n program_id: Program ID\n\n Returns:\n Artifacts dictionary or None if not found\n \"\"\"\n return self._pending_artifacts.pop(program_id, None)\n\n async def _direct_evaluate(self, program_path: str) -> Dict[str, float]:\n \"\"\"\n Directly evaluate a program using the evaluation function with timeout\n\n Args:\n program_path: Path to the program file\n\n Returns:\n Dictionary of metric name to score\n\n Raises:\n asyncio.TimeoutError: If evaluation exceeds timeout\n Exception: If evaluation function raises an exception\n \"\"\"\n\n # Create a coroutine that runs the evaluation function in an executor\n async def run_evaluation():\n loop = asyncio.get_event_loop()\n return await loop.run_in_executor(None, self.evaluate_function, program_path)\n\n # Run the evaluation with timeout - let exceptions bubble up for retry handling\n result = await asyncio.wait_for(run_evaluation(), timeout=self.config.timeout)\n\n # Validate result\n if not isinstance(result, dict):\n logger.warning(f\"Evaluation returned non-dictionary result: {result}\")\n return {\"error\": -1.0}\n\n return result\n\n async def _cascade_evaluate(\n self, program_path: str\n ) -> Union[Dict[str, float], EvaluationResult]:\n \"\"\"\n Run cascade evaluation with increasingly challenging test cases\n\n Args:\n program_path: Path to the program file\n\n Returns:\n Dictionary of metrics or EvaluationResult with metrics and artifacts\n \"\"\"\n # Import the evaluation module to get cascade functions if they exist\n try:\n # Add the evaluation file's directory to Python path so it can import local modules\n eval_dir = os.path.dirname(os.path.abspath(self.evaluation_file))\n if eval_dir not in sys.path:\n sys.path.insert(0, eval_dir)\n logger.debug(f\"Added {eval_dir} to Python path for cascade evaluation\")\n\n spec = importlib.util.spec_from_file_location(\"evaluation_module\", self.evaluation_file)\n if spec is None or spec.loader is None:\n return await self._direct_evaluate(program_path)\n\n module = importlib.util.module_from_spec(spec)\n spec.loader.exec_module(module)\n\n # Check if cascade functions exist\n if not hasattr(module, \"evaluate_stage1\"):\n return await self._direct_evaluate(program_path)\n\n # Run first stage with timeout\n try:\n async def run_stage1():\n loop = asyncio.get_event_loop()\n return await loop.run_in_executor(None, module.evaluate_stage1, program_path)\n\n stage1_result = await asyncio.wait_for(run_stage1(), timeout=self.config.timeout)\n stage1_eval_result = self._process_evaluation_result(stage1_result)\n except asyncio.TimeoutError:\n logger.warning(f\"Stage 1 evaluation timed out after {self.config.timeout}s\")\n return EvaluationResult(\n metrics={\"stage1_passed\": -1.0, \"error\": -1.0, \"timeout\": True},\n artifacts={\n \"failure_stage\": \"stage1\",\n \"timeout\": True,\n },\n )\n except Exception as e:\n logger.error(f\"Error in stage 1 evaluation: {str(e)}\")\n # Capture stage 1 failure as artifacts\n return EvaluationResult(\n metrics={\"stage1_passed\": -1.0, \"error\": -1.0},\n artifacts={\n \"stderr\": str(e),\n \"traceback\": traceback.format_exc(),\n \"failure_stage\": \"stage1\",\n },\n )\n\n # Check threshold\n if not self._passes_threshold(\n stage1_eval_result.metrics, self.config.cascade_thresholds[0]\n ):\n return stage1_eval_result\n\n # Check if second stage exists\n if not hasattr(module, \"evaluate_stage2\"):\n return stage1_eval_result\n\n # Run second stage with timeout\n try:\n\n async def run_stage2():\n loop = asyncio.get_event_loop()\n return await loop.run_in_executor(None, module.evaluate_stage2, program_path)\n\n stage2_result = await asyncio.wait_for(run_stage2(), timeout=self.config.timeout)\n stage2_eval_result = self._process_evaluation_result(stage2_result)\n except asyncio.TimeoutError:\n logger.warning(f\"Stage 2 evaluation timed out after {self.config.timeout}s\")\n # Capture stage 2 failure, but keep stage 1 results\n stage1_eval_result.artifacts.update(\n {\n \"stage2_timeout\": True,\n \"failure_stage\": \"stage2\",\n }\n )\n stage1_eval_result.metrics[\"stage2_passed\"] = 0.0\n stage1_eval_result.metrics[\"timeout\"] = True\n return stage1_eval_result\n except Exception as e:\n logger.error(f\"Error in stage 2 evaluation: {str(e)}\")\n # Capture stage 2 failure, but keep stage 1 results\n stage1_eval_result.artifacts.update(\n {\n \"stage2_stderr\": str(e),\n \"stage2_traceback\": traceback.format_exc(),\n \"failure_stage\": \"stage2\",\n }\n )\n stage1_eval_result.metrics[\"stage2_passed\"] = 0.0\n return stage1_eval_result\n\n # Merge results from stage 1 and 2\n merged_metrics = {}\n # Convert all values to float to avoid type errors\n for name, value in stage1_eval_result.metrics.items():\n if isinstance(value, (int, float)) and name != \"error\":\n merged_metrics[name] = float(value)\n\n for name, value in stage2_eval_result.metrics.items():\n if isinstance(value, (int, float)) and name != \"error\":\n merged_metrics[name] = float(value)\n\n # Merge artifacts\n merged_artifacts = {}\n merged_artifacts.update(stage1_eval_result.artifacts)\n merged_artifacts.update(stage2_eval_result.artifacts)\n\n merged_result = EvaluationResult(metrics=merged_metrics, artifacts=merged_artifacts)\n\n # Check threshold for stage 3\n if len(self.config.cascade_thresholds) < 2 or not self._passes_threshold(\n merged_result.metrics, self.config.cascade_thresholds[1]\n ):\n return merged_result\n\n # Check if third stage exists\n if not hasattr(module, \"evaluate_stage3\"):\n return merged_result\n\n # Run third stage with timeout\n try:\n\n async def run_stage3():\n loop = asyncio.get_event_loop()\n return await loop.run_in_executor(None, module.evaluate_stage3, program_path)\n\n stage3_result = await asyncio.wait_for(run_stage3(), timeout=self.config.timeout)\n stage3_eval_result = self._process_evaluation_result(stage3_result)\n except asyncio.TimeoutError:\n logger.warning(f\"Stage 3 evaluation timed out after {self.config.timeout}s\")\n # Capture stage 3 failure, but keep previous results\n merged_result.artifacts.update(\n {\n \"stage3_timeout\": True,\n \"failure_stage\": \"stage3\",\n }\n )\n merged_result.metrics[\"stage3_passed\"] = 0.0\n merged_result.metrics[\"timeout\"] = True\n return merged_result\n except Exception as e:\n logger.error(f\"Error in stage 3 evaluation: {str(e)}\")\n # Capture stage 3 failure, but keep previous results\n merged_result.artifacts.update(\n {\n \"stage3_stderr\": str(e),\n \"stage3_traceback\": traceback.format_exc(),\n \"failure_stage\": \"stage3\",\n }\n )\n merged_result.metrics[\"stage3_passed\"] = 0.0\n return merged_result\n\n # Merge stage 3 results\n for name, value in stage3_eval_result.metrics.items():\n if isinstance(value, (int, float)) and name != \"error\":\n merged_result.metrics[name] = float(value)\n\n merged_result.artifacts.update(stage3_eval_result.artifacts)\n\n return merged_result\n\n except Exception as e:\n logger.error(f\"Error in cascade evaluation: {str(e)}\")\n # Return proper cascade failure result instead of re-raising\n return EvaluationResult(\n metrics={\"stage1_passed\": -1.0, \"error\": -1.0},\n artifacts={\n \"stderr\": str(e),\n \"traceback\": traceback.format_exc(),\n \"failure_stage\": \"cascade_setup\",\n },\n )\n\n async def _llm_evaluate(self, program_code: str, program_id: str = \"\") -> Dict[str, float]:\n \"\"\"\n Use LLM to evaluate code quality\n\n Args:\n program_code: Code to evaluate\n program_id: Optional ID for logging\n\n Returns:\n Dictionary of metric name to score\n \"\"\"\n if not self.llm_ensemble:\n return {}\n\n try:\n # Create prompt for LLM\n prompt = self.prompt_sampler.build_prompt(\n current_program=program_code, template_key=\"evaluation\"\n )\n\n # Get LLM response\n responses = await self.llm_ensemble.generate_all_with_context(\n prompt[\"system\"], [{\"role\": \"user\", \"content\": prompt[\"user\"]}]\n )\n\n # Log prompt and response to database\n if self.database and program_id:\n self.database.log_prompt(\n program_id=program_id,\n template_key=\"evaluation\",\n prompt=prompt,\n responses=responses,\n )\n\n # Extract JSON from response\n try:\n # Try to find JSON block\n json_pattern = r\"```json\\n(.*?)\\n```\"\n import re\n\n artifacts = {}\n avg_metrics = {}\n for i, response in enumerate(responses):\n json_match = re.search(json_pattern, response, re.DOTALL)\n\n if json_match:\n json_str = json_match.group(1)\n else:\n # Try to extract JSON directly\n json_str = response\n # Remove non-JSON parts\n start_idx = json_str.find(\"{\")\n end_idx = json_str.rfind(\"}\") + 1\n if start_idx >= 0 and end_idx > start_idx:\n json_str = json_str[start_idx:end_idx]\n\n # Parse JSON\n result = json.loads(json_str)\n\n # All non-numeric values are artifacts, all numeric values are metrics\n metrics = {}\n for key, value in result.items():\n if not isinstance(value, (int, float)):\n artifacts[key] = value\n else:\n metrics[key] = float(value)\n\n # Weight of the model in the ensemble\n weight = self.llm_ensemble.weights[i] if self.llm_ensemble.weights else 1.0\n\n # Average the metrics\n for name, value in metrics.items():\n if name in avg_metrics:\n avg_metrics[name] += value * weight\n else:\n avg_metrics[name] = value * weight\n\n return EvaluationResult(\n metrics=avg_metrics,\n artifacts=artifacts,\n )\n\n except Exception as e:\n logger.warning(f\"Error parsing LLM response: {str(e)}\")\n return {}\n\n except Exception as e:\n logger.error(f\"Error in LLM evaluation: {str(e)}\")\n traceback.print_exc()\n return {}\n\n def _passes_threshold(self, metrics: Dict[str, float], threshold: float) -> bool:\n \"\"\"\n Check if metrics pass a threshold\n\n Args:\n metrics: Dictionary of metric name to score\n threshold: Threshold to pass\n\n Returns:\n True if metrics pass threshold\n \"\"\"\n if not metrics:\n return False\n\n # Calculate average score, skipping non-numeric values and 'error' key\n valid_metrics = []\n for name, value in metrics.items():\n # Skip 'error' keys and ensure values are numeric\n if name != \"error\" and isinstance(value, (int, float)):\n try:\n valid_metrics.append(float(value))\n except (TypeError, ValueError):\n logger.warning(f\"Skipping non-numeric metric: {name}={value}\")\n continue\n\n if not valid_metrics:\n return False\n\n avg_score = sum(valid_metrics) / len(valid_metrics)\n return avg_score >= threshold\n\n async def evaluate_multiple(\n self,\n programs: List[Tuple[str, str]],\n ) -> List[Dict[str, float]]:\n \"\"\"\n Evaluate multiple programs in parallel\n\n Args:\n programs: List of (program_code, program_id) tuples\n\n Returns:\n List of metric dictionaries\n \"\"\"\n tasks = [\n self.task_pool.create_task(self.evaluate_program, program_code, program_id)\n for program_code, program_id in programs\n ]\n\n return await asyncio.gather(*tasks)\n"}}, "benchmark/spherical_code/visualization.py": {"174553854fc8182945432761538118f20b7f072e": {"t": 1785599212.871936, "n": 3706, "h": "3990a447e0f8cd5d", "c": "# viz_sphere_points.py\nimport argparse\nimport importlib.util\nimport sys\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D # noqa: F401 (needed for 3D)\n\ndef _normalize_rows(P):\n nrm = np.linalg.norm(P, axis=1, keepdims=True)\n nrm = np.maximum(nrm, 1e-12)\n return P / nrm\n\ndef load_from_module(module_path: str):\n \"\"\"\n 动态加载模块：\n - 优先读取全局变量 `points`\n - 否则调用 `main()` 获取返回值\n \"\"\"\n module_path = os.path.abspath(module_path)\n spec = importlib.util.spec_from_file_location(\"points_mod\", module_path)\n if spec is None or spec.loader is None:\n raise RuntimeError(f\"无法加载模块: {module_path}\")\n mod = importlib.util.module_from_spec(spec)\n spec.loader.exec_module(mod)\n\n if hasattr(mod, \"points\"):\n pts = getattr(mod, \"points\")\n elif hasattr(mod, \"main\"):\n pts = mod.main()\n else:\n raise RuntimeError(\"模块中既无 `points` 变量，也无 `main()` 函数可获取点。\")\n\n pts = np.asarray(pts, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 3:\n raise ValueError(f\"模块返回的点形状异常: {pts.shape}, 期望 (N, 3)\")\n return _normalize_rows(pts)\n\ndef load_from_npy(npy_path: str):\n pts = np.load(npy_path)\n pts = np.asarray(pts, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 3:\n raise ValueError(f\"npy 形状异常: {pts.shape}, 期望 (N, 3)\")\n return _normalize_rows(pts)\n\ndef load_from_csv(csv_path: str):\n pts = np.loadtxt(csv_path, delimiter=\",\")\n pts = np.asarray(pts, dtype=float)\n if pts.ndim != 2 or pts.shape[1] != 3:\n raise ValueError(f\"csv 形状异常: {pts.shape}, 期望 (N, 3)\")\n return _normalize_rows(pts)\n\ndef min_pairwise_angle_deg(P):\n \"\"\"\n 返回：\n - 最小夹角（度）\n - 对应的最大余弦相似度（最近的一对点）\n \"\"\"\n # 计算上三角的点积\n dot = P @ P.T\n n = len(P)\n mask = np.triu(np.ones((n, n), dtype=bool), k=1)\n vals = dot[mask]\n max_cos = np.max(vals) if vals.size else 1.0\n max_cos = np.clip(max_cos, -1.0, 1.0)\n ang_min = np.degrees(np.arccos(max_cos))\n return ang_min, max_cos\n\ndef plot_points_on_sphere(P, title=\"Spherical Point Set\"):\n fig = plt.figure(figsize=(7, 7))\n ax = fig.add_subplot(111, projection=\"3d\")\n ax.set_box_aspect([1,1,1])\n\n # 画单位球面网格\n u = np.linspace(0, 2*np.pi, 60)\n v = np.linspace(0, np.pi, 30)\n x = np.outer(np.cos(u), np.sin(v))\n y = np.outer(np.sin(u), np.sin(v))\n z = np.outer(np.ones_like(u), np.cos(v))\n ax.plot_surface(x, y, z, alpha=0.15, linewidth=0, antialiased=True)\n\n # 画点\n ax.scatter(P[:,0], P[:,1], P[:,2], s=40, depthshade=True)\n\n # 坐标与视角\n ax.set_xlabel(\"x\")\n ax.set_ylabel(\"y\")\n ax.set_zlabel(\"z\")\n ax.set_title(title)\n ax.view_init(elev=20, azim=45)\n\n plt.tight_layout()\n plt.show()\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"可视化 S^2 上的点集（从模块、.npy 或 .csv 读取）\"\n )\n src = parser.add_mutually_exclusive_group(required=True)\n src.add_argument(\"--from-module\", type=str, help=\"包含 points 或 main() 的 .py 文件路径（例如 sphere_points.py）\")\n src.add_argument(\"--from-npy\", type=str, help=\"N×3 的 .npy 文件路径\")\n src.add_argument(\"--from-csv\", type=str, help=\"N×3 的 .csv 文件路径（逗号分隔）\")\n parser.add_argument(\"--title\", type=str, default=\"Spherical Point Set\", help=\"图标题\")\n args = parser.parse_args()\n\n if args.from_module:\n P = load_from_module(args.from_module)\n elif args.from_npy:\n P = load_from_npy(args.from_npy)\n else:\n P = load_from_csv(args.from_csv)\n\n ang_min_deg, max_cos = min_pairwise_angle_deg(P)\n print(f\"N = {len(P)}\")\n print(f\"最小两点夹角 ≈ {ang_min_deg:.4f}°\")\n print(f\"最近对的余弦相似度（最大点积） ≈ {max_cos:.6f}\")\n\n plot_points_on_sphere(P, title=args.title)\n\nif __name__ == \"__main__\":\n main()\n"}}, "evolve_agent/evaluation_result.py": {"382b77e774378f18411e87bb7d5f08a725b21750": {"t": 1785599213.1895618, "n": 1779, "h": "afb4723bf28b9b1c", "c": "\"\"\"\nEvaluation result structures for EvolveAgent\n\"\"\"\n\nimport json\nfrom dataclasses import dataclass, field\nfrom typing import Dict, Union\n\n\n@dataclass\nclass EvaluationResult:\n \"\"\"\n Result of program evaluation containing both metrics and optional artifacts\n\n This maintains backward compatibility with the existing dict[str, float] contract\n while adding a side-channel for arbitrary artifacts (text or binary data).\n \"\"\"\n\n metrics: Dict[str, float] # mandatory - existing contract\n artifacts: Dict[str, Union[str, bytes]] = field(default_factory=dict) # optional side-channel\n\n @classmethod\n def from_dict(cls, metrics: Dict[str, float]) -> \"EvaluationResult\":\n \"\"\"Auto-wrap dict returns for backward compatibility\"\"\"\n return cls(metrics=metrics)\n\n def to_dict(self) -> Dict[str, float]:\n \"\"\"Backward compatibility - return just metrics\"\"\"\n return self.metrics\n\n def has_artifacts(self) -> bool:\n \"\"\"Check if this result contains any artifacts\"\"\"\n return bool(self.artifacts)\n\n def get_artifact_keys(self) -> list:\n \"\"\"Get list of artifact keys\"\"\"\n return list(self.artifacts.keys())\n\n def get_artifact_size(self, key: str) -> int:\n \"\"\"Get size of a specific artifact in bytes\"\"\"\n if key not in self.artifacts:\n return 0\n\n value = self.artifacts[key]\n if isinstance(value, str):\n return len(value.encode(\"utf-8\"))\n elif isinstance(value, bytes):\n return len(value)\n else:\n return len(str(value).encode(\"utf-8\"))\n\n def get_total_artifact_size(self) -> int:\n \"\"\"Get total size of all artifacts in bytes\"\"\"\n return sum(self.get_artifact_size(key) for key in self.artifacts.keys())\n"}}, "benchmark/packing_circles/initial_program.py": {"3b2fdb152eaa01c95dcd5507999f7faba488dd15": {"t": 1785599213.2147105, "n": 14255, "h": "666b3d9b61f282aa", "c": "import math\nimport random\nfrom concurrent.futures import ThreadPoolExecutor\n\n\ndef pack_circles(n, square_size=1.0):\n \"\"\"\n Pack n disjoint circles in a unit square using uniform tiling approach.\n Returns the sum of radii and list of circles (x, y, r).\n \"\"\"\n\n def max_circle_radius(x, y, circles, square_size=1.0, skip_idx=None):\n \"\"\"\n Compute the maximum radius for a circle centered at (x, y) that:\n - Stays within the unit square [0, square_size] × [0, square_size].\n - Does not overlap with existing circles.\n skip_idx: if provided, index in circles[] to ignore (self).\n \"\"\"\n # Distance to nearest boundary of the unit square\n r_max = min(x, y, square_size - x, square_size - y)\n \n # Check distance to existing circles, exit early if r_max → 0\n # early exit if r_max is tiny, and avoid needless sqrt\n for idx, (cx, cy, cr) in enumerate(circles):\n if skip_idx == idx:\n continue\n if r_max <= 1e-8:\n break\n dx = x - cx\n dy = y - cy\n sep = r_max + cr\n if dx*dx + dy*dy < sep*sep:\n # only compute sqrt when we know we can shrink\n dist = math.sqrt(dx*dx + dy*dy)\n r_max = min(r_max, dist - cr)\n return max(r_max, 0.0)\n\n def uniform_tiling_circles(n, square_size=1.0):\n \"\"\"\n Uniformly tile the square with circles using optimal grid placement.\n \"\"\"\n if n <= 0:\n return []\n \n circles = []\n \n # Calculate optimal grid dimensions\n # For n circles, find the best grid layout (rows x cols)\n best_layout = None\n best_total_radius = 0\n \n # Try different grid configurations\n for rows in range(1, min(n + 1, 20)):\n cols = math.ceil(n / rows)\n if cols > 20: # Limit grid size\n continue\n \n # Calculate spacing\n spacing_x = square_size / (cols + 1)\n spacing_y = square_size / (rows + 1)\n \n # Use the smaller spacing to ensure circles fit\n min_spacing = min(spacing_x, spacing_y)\n \n # Calculate maximum radius for this layout\n max_radius = min_spacing / 2\n \n # Ensure radius doesn't exceed boundaries\n max_radius = min(max_radius, \n spacing_x / 2 - 1e-6, \n spacing_y / 2 - 1e-6)\n \n if max_radius <= 0:\n continue\n \n # Place circles in uniform grid\n temp_circles = []\n count = 0\n \n for row in range(rows):\n for col in range(cols):\n if count >= n:\n break\n \n x = spacing_x * (col + 1)\n y = spacing_y * (row + 1)\n \n # Ensure circle stays within bounds\n if (x - max_radius >= 0 and x + max_radius <= square_size and\n y - max_radius >= 0 and y + max_radius <= square_size):\n \n temp_circles.append((x, y, max_radius))\n count += 1\n \n if count >= n:\n break\n \n # Calculate total radius for this layout\n total_radius = len(temp_circles) * max_radius\n \n if total_radius > best_total_radius and len(temp_circles) == n:\n best_total_radius = total_radius\n best_layout = temp_circles\n \n # If we found a valid layout, return it\n if best_layout:\n return best_layout\n \n # Fallback: use hexagonal packing for better density\n return hexagonal_packing(n, square_size)\n\n def hexagonal_packing(n, square_size=1.0):\n \"\"\"\n Use hexagonal close packing for better space utilization.\n \"\"\"\n circles = []\n \n # Estimate number of rows and columns for hexagonal packing\n # Hexagonal packing has rows offset by sqrt(3)/2 * diameter\n \n rows = int(math.sqrt(n * 2 / math.sqrt(3))) + 2\n \n count = 0\n row = 0\n \n while count < n and row < rows:\n # Calculate y position for this row\n y = (row + 0.5) * (square_size / (rows + 1))\n \n # Number of circles in this row\n if row % 2 == 0:\n cols = int(math.sqrt(n)) + 1\n else:\n cols = int(math.sqrt(n))\n \n spacing_x = square_size / (cols + 1)\n \n for col in range(cols):\n if count >= n:\n break\n \n if row % 2 == 0:\n x = spacing_x * (col + 1)\n else:\n x = spacing_x * (col + 1) + spacing_x / 2\n \n # Calculate maximum radius for this position\n r = max_circle_radius(x, y, circles, square_size)\n \n if r > 0:\n circles.append((x, y, r))\n count += 1\n \n row += 1\n \n return circles\n\n def optimize_placement(n, square_size=1.0):\n \"\"\"\n Optimize circle placement using uniform tiling with radius maximization.\n \"\"\"\n circles = []\n \n # First, try hexagonal packing for high initial density\n hex_circles = hexagonal_packing(n, square_size)\n if len(hex_circles) == n:\n # Ensure maximum radii for hex layout with stronger refinement\n hex_refined = refine_circles(hex_circles, square_size, iterations=20)\n return hex_refined\n \n # Fallback to uniform grid placement\n grid_circles = uniform_tiling_circles(n, square_size)\n if len(grid_circles) == n:\n return grid_circles\n \n # If uniform tiling didn't work perfectly, use adaptive approach\n # Calculate optimal radius based on density\n area_per_circle = (square_size * square_size) / n\n estimated_radius = math.sqrt(area_per_circle / math.pi) * 0.9 # Conservative estimate\n \n # Create grid with optimal spacing\n spacing = estimated_radius * 2.1 # Include gap\n \n cols = int(square_size / spacing)\n rows = int(square_size / spacing)\n \n actual_spacing_x = square_size / (cols + 1)\n actual_spacing_y = square_size / (rows + 1)\n \n count = 0\n for row in range(rows):\n for col in range(cols):\n if count >= n:\n break\n \n x = actual_spacing_x * (col + 1)\n y = actual_spacing_y * (row + 1)\n \n # Calculate maximum possible radius\n r = max_circle_radius(x, y, circles, square_size)\n \n if r > 0:\n circles.append((x, y, r))\n count += 1\n \n if count >= n:\n break\n \n # If we still need more circles, use remaining space\n remaining = n - len(circles)\n if remaining > 0:\n # Place remaining circles in remaining spaces\n for i in range(remaining):\n # Try different positions systematically\n best_r = 0\n best_pos = (0.5, 0.5)\n \n # Fine grid search (increased resolution)\n grid_points = 100\n for gx in range(1, grid_points):\n for gy in range(1, grid_points):\n x = gx / grid_points\n y = gy / grid_points\n \n r = max_circle_radius(x, y, circles, square_size)\n if r > best_r:\n best_r = r\n best_pos = (x, y)\n \n if best_r > 0:\n circles.append((best_pos[0], best_pos[1], best_r))\n \n return circles\n\n def refine_circles(circles, square_size, iterations=80, perturb_interval=3):\n \"\"\"\n Iteratively grow each circle to its maximum radius under non-overlap constraints.\n Includes randomized update order, periodic micro-perturbation to escape\n local minima, and a final local-center-perturbation pass for densification.\n \"\"\"\n for it in range(iterations):\n # randomize update order to avoid sweep-order bias\n indices = list(range(len(circles)))\n random.shuffle(indices)\n for i in indices:\n x, y, _ = circles[i]\n # Compute maximal feasible radius here, skipping self\n r = max_circle_radius(x, y, circles, square_size, skip_idx=i)\n circles[i] = (x, y, r)\n # Periodic micro-perturbation: jiggle a few circles\n if it % perturb_interval == 0 and len(circles) > 0:\n subset = random.sample(indices, min(5, len(circles)))\n for j in subset:\n x0, y0, r0 = circles[j]\n dx = random.uniform(-0.03, 0.03)\n dy = random.uniform(-0.03, 0.03)\n nx = min(max(x0 + dx, 0), square_size)\n ny = min(max(y0 + dy, 0), square_size)\n # Compute maximal radius skipping self\n nr = max_circle_radius(nx, ny, circles, square_size, skip_idx=j)\n if nr > r0:\n circles[j] = (nx, ny, nr)\n # Full local center-perturbation phase for final densification\n for i in range(len(circles)):\n x, y, r = circles[i]\n best_x, best_y, best_r = x, y, r\n delta = 0.1\n for _ in range(20):\n dx = random.uniform(-delta, delta)\n dy = random.uniform(-delta, delta)\n nx = min(max(x + dx, 0), square_size)\n ny = min(max(y + dy, 0), square_size)\n # Compute maximal radius skipping self\n nr = max_circle_radius(nx, ny, circles, square_size, skip_idx=i)\n if nr > best_r:\n best_x, best_y, best_r = nx, ny, nr\n else:\n delta *= 0.9\n circles[i] = (best_x, best_y, best_r)\n \n # Physics-inspired soft relaxation to escape persistent overlaps\n for i in range(len(circles)):\n x, y, r = circles[i]\n fx, fy = 0.0, 0.0\n for j, (xj, yj, rj) in enumerate(circles):\n if i == j:\n continue\n dx = x - xj\n dy = y - yj\n d = (dx*dx + dy*dy) ** 0.5\n overlap = (r + rj) - d\n if overlap > 0 and d > 1e-8:\n fx += dx / d * overlap\n fy += dy / d * overlap\n # Nudge the center by 10% of the computed net “repulsive” force\n nx = min(max(x + 0.1 * fx, 0), square_size)\n ny = min(max(y + 0.1 * fy, 0), square_size)\n nr = max_circle_radius(nx, ny, circles, square_size, skip_idx=i)\n circles[i] = (nx, ny, nr)\n return circles\n\n def multi_start_optimize(n, square_size, starts=None):\n \"\"\"\n Parallel multi-start global → local optimization using ThreadPoolExecutor.\n Number of starts adapts to problem size: max(100, 10*n).\n \"\"\"\n if starts is None:\n if n <= 50:\n starts = max(200, n * 20)\n else:\n starts = max(100, n * 10)\n # precompute hexagonal‐packing baseline\n hex_circ = hexagonal_packing(n, square_size)\n hex_sum = sum(r for _, _, r in hex_circ)\n best_conf = None\n best_sum = 0.0\n\n # single trial: seed → refine → score\n def single_run(_):\n conf0 = optimize_placement(n, square_size)\n conf1 = refine_circles(conf0, square_size, iterations=40)\n s1 = sum(r for _, _, r in conf1)\n return s1, conf1\n\n # dispatch trials in parallel\n with ThreadPoolExecutor() as executor:\n for score, conf in executor.map(single_run, range(starts)):\n if score > best_sum:\n best_sum, best_conf = score, conf.copy()\n # early exit if near the hex-baseline\n if best_sum >= hex_sum * 0.995:\n break\n\n return best_conf\n\n # Use multi-start global → local optimization (adaptive number of starts)\n circles = multi_start_optimize(n, square_size)\n\n # Quick 2-cluster remove-and-reinsert densification (extended iterations)\n for _ in range(8):\n # remove the two smallest circles to create a larger gap\n smallest = sorted(range(len(circles)), key=lambda i: circles[i][2])[:2]\n removed = [circles[i] for i in smallest]\n # pop in reverse order to keep indices valid\n for i in sorted(smallest, reverse=True):\n circles.pop(i)\n # refine the remaining configuration briefly\n circles = refine_circles(circles, square_size, iterations=8)\n # reinsert each removed circle with more sampling\n for x_old, y_old, _ in removed:\n best_r, best_pos = 0.0, (x_old, y_old)\n for _ in range(500):\n x = random.uniform(0, square_size)\n y = random.uniform(0, square_size)\n r = max_circle_radius(x, y, circles, square_size)\n if r > best_r:\n best_r, best_pos = r, (x, y)\n circles.append((best_pos[0], best_pos[1], best_r))\n # final local polish after reinsertion\n circles = refine_circles(circles, square_size, iterations=5)\n # end 2-cluster remove-and-reinsert densification\n\n # Calculate total radius\n total_radius = sum(circle[2] for circle in circles)\n \n return total_radius, circles"}}, "evolve_agent/llm/base.py": {"1cbf436e95845a30fff030a9dda2420cf9a69e50": {"t": 1785599214.254338, "n": 560, "h": "1bcbc1a9b3e89e27", "c": "\"\"\"\nBase LLM interface\n\"\"\"\n\nfrom abc import ABC, abstractmethod\nfrom typing import Any, Dict, List, Optional\n\n\nclass LLMInterface(ABC):\n \"\"\"Abstract base class for LLM interfaces\"\"\"\n\n @abstractmethod\n async def generate(self, prompt: str, **kwargs) -> str:\n \"\"\"Generate text from a prompt\"\"\"\n pass\n\n @abstractmethod\n async def generate_with_context(\n self, system_message: str, messages: List[Dict[str, str]], **kwargs\n ) -> str:\n \"\"\"Generate text using a system message and conversational context\"\"\"\n pass\n"}}, "evolve_agent/reward_model.py": {"05a2f3bb1f9362321a006ef6543b25a0610ab975": {"t": 1785599214.4011476, "n": 12573, "h": "2b0b60d0c8e66a58", "c": "import re\nimport os\nimport json\nimport time\nfrom typing import List, Dict, Optional, Any\nfrom pathlib import Path\n\nimport asyncio\n# import aiofiles\nimport numpy as np\nimport pandas as pd\nfrom vllm import LLM, SamplingParams\nfrom openai import AsyncOpenAI\nfrom transformers import AutoTokenizer\nfrom datasets import load_dataset\n\nfrom evolve_agent.config import RewardModelConfig\n\n\nclass RewardModel:\n \"\"\"\n A class to score research abstracts or proposals based on clarity, novelty, technical rigor,\n and potential impact using either a local vLLM model or an external API model.\n \"\"\"\n \n BOX = r\"\\boxed{}\"\n SYSTEM_PROMPT = \"You are an expert reviewer tasked with evaluating the quality of a research proposal.\"\n SCORING_PROMPT = f\"\"\"\nYour goal is to assign a score between 1 and 10 based on the proposal's clarity, novelty, technical rigor, and potential impact. Here are the criteria:\n1. Read the following proposal carefully and provide a score from 1 to 10. \n2. Score 6 means slightly higher than the borderline, 5 is slightly lower than the borderline.\nWrite the score in the {BOX}.\n**idea**:\n\"\"\"\n\n def __init__(self, config: RewardModelConfig):\n \"\"\"\n Initialize the RewardModel.\n\n Args:\n config (RewardModelConfig): Configuration object containing model_type, model_name, api_key, base_url,\n jsonl_file, max_retries, retry_delay, temperature, top_p, max_tokens.\n \"\"\"\n self.config = config\n\n if self.config.model_type == \"vllm\":\n self.llm = LLM(model=self.config.model_name, gpu_memory_utilization=0.95)\n self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_name)\n elif self.config.model_type == \"api\":\n if not self.config.api_key or not self.config.base_url:\n raise ValueError(\"API key and base URL must be provided for API model type.\")\n self.client = AsyncOpenAI(api_key=self.config.api_key, base_url=self.config.base_url)\n else:\n raise ValueError(\"model_type must be 'vllm' or 'api'.\")\n\n # Ensure the directory for jsonl_file exists\n os.makedirs(os.path.dirname(self.config.jsonl_file) or \".\", exist_ok=True)\n\n def parse_score_from_text(self, text: str) -> float:\n \"\"\"\n Parse the score from the model's output text.\n\n Args:\n text (str): Model output containing the score in \\boxed{} format.\n\n Returns:\n float: Parsed score between 0 and 10, or -1.0 if invalid.\n \"\"\"\n match = re.search(r'\\\\boxed\\{(\\d*\\.?\\d*)\\}', text)\n if match:\n try:\n score = float(match.group(1))\n if 0 <= score <= 10:\n return score\n except ValueError:\n pass\n return -1.0\n\n async def load_processed_ids(self) -> set:\n \"\"\"\n Load titles of already processed abstracts from the JSONL file.\n\n Returns:\n set: Set of processed abstract titles with valid scores.\n \"\"\"\n processed_ids = set()\n # if os.path.exists(self.config.jsonl_file):\n # async with aiofiles.open(self.config.jsonl_file, 'r', encoding='utf-8') as f:\n # async for line in f:\n # try:\n # data = json.loads(line.strip())\n # if data.get('score', -1.0) != -1.0:\n # processed_ids.add(data['title'])\n # except json.JSONDecodeError:\n # print(f\"Warning: Skipping invalid JSON line in {self.config.jsonl_file}\")\n return processed_ids\n\n async def write_result_to_jsonl(self, result: Dict):\n \"\"\"\n Write a single result to the JSONL file if the score is valid.\n\n Args:\n result (Dict): Result dictionary containing title, score, evaluation, abstract, and gt_score.\n \"\"\"\n # if result['score'] != -1.0:\n # async with aiofiles.open(self.config.jsonl_file, 'a', encoding='utf-8') as f:\n # await f.write(json.dumps(result, ensure_ascii=False) + '\\n')\n\n async def score_with_vllm(self, data: List[Dict]) -> List[Dict]:\n \"\"\"\n Score abstracts using a local vLLM model.\n\n Args:\n data (List[Dict]): List of dictionaries containing 'title', 'abstract', and 'gt_score'.\n\n Returns:\n List[Dict]: List of results with 'title', 'score', 'evaluation', 'abstract', and 'gt_score'.\n \"\"\"\n prompts = [\n self.tokenizer.apply_chat_template(\n [\n {\"role\": \"system\", \"content\": self.SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": self.SCORING_PROMPT + item[\"title\"] + \"\\n\" + item[\"abstract\"]}\n ],\n tokenize=False,\n add_generation_prompt=True,\n enable_thinking=False\n )\n for item in data\n ]\n\n sampling_params = SamplingParams(\n temperature=self.config.temperature,\n top_p=self.config.top_p,\n max_tokens=self.config.max_tokens,\n )\n\n # vLLM is synchronous, so we run it in the default executor\n outputs = await asyncio.get_event_loop().run_in_executor(None, lambda: self.llm.generate(prompts, sampling_params))\n\n results = []\n for output, item in zip(outputs, data):\n output_text = output.outputs[0].text.strip()\n score = self.parse_score_from_text(output_text)\n result = {\n \"title\": item[\"title\"],\n \"score\": score,\n \"evaluation\": output_text,\n \"abstract\": item[\"abstract\"],\n \"gt_score\": item[\"gt_score\"]\n }\n await self.write_result_to_jsonl(result)\n results.append(result)\n\n return results\n\n async def score_with_api(self, data: List[Dict]) -> List[Dict]:\n \"\"\"\n Score abstracts using an external API model.\n\n Args:\n data (List[Dict]): List of dictionaries containing 'title', 'abstract', and 'gt_score'.\n\n Returns:\n List[Dict]: List of results with 'title', 'score', 'evaluation', 'abstract', and 'gt_score'.\n \"\"\"\n processed_ids = await self.load_processed_ids()\n data_to_process = [item for item in data if item['title'] not in processed_ids]\n print(f\"Total abstracts: {len(data)}, To process: {len(data_to_process)}, Already processed: {len(processed_ids)}\")\n\n prompts = [\n [\n {\"role\": \"system\", \"content\": self.SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": self.SCORING_PROMPT + item[\"title\"] + \"\\n\" + item[\"abstract\"]}\n ]\n for item in data_to_process\n ]\n\n results = []\n for prompt, item in zip(prompts, data_to_process):\n retries = 0\n score = -1.0\n output_text = \"\"\n\n while score == -1.0 and retries < self.config.max_retries:\n try:\n response = await self.client.chat.completions.create(\n model=self.config.model_name,\n messages=prompt,\n temperature=0, # API uses fixed temperature as per original code\n max_tokens=1000, # API uses fixed max_tokens as per original code\n top_p=1.0 # API uses fixed top_p as per original code\n )\n output_text = response.choices[0].message.content.strip()\n score = self.parse_score_from_text(output_text)\n except Exception as e:\n print(f\"Error processing {item['title']}: {e}\")\n \n if score == -1.0:\n retries += 1\n print(f\"Invalid score for abstract: {item['title']}, Retry {retries}/{self.config.max_retries}\")\n await asyncio.sleep(self.config.retry_delay)\n\n result = {\n \"title\": item[\"title\"],\n \"score\": score,\n \"gt_score\": item[\"gt_score\"],\n \"evaluation\": output_text,\n \"abstract\": item[\"abstract\"]\n }\n await self.write_result_to_jsonl(result)\n results.append(result)\n\n if score == -1.0:\n print(f\"Failed to get valid score for abstract: {item['title']} after {self.config.max_retries} retries\")\n\n # Include previously processed results\n # if processed_ids:\n # async with aiofiles.open(self.config.jsonl_file, 'r', encoding='utf-8') as f:\n # async for line in f:\n # try:\n # result = json.loads(line.strip())\n # if result['title'] in processed_ids:\n # results.append(result)\n # except json.JSONDecodeError:\n # print(f\"Warning: Skipping invalid JSON line in {self.config.jsonl_file}\")\n\n return results\n\n async def score_research_proposal(self, data: List[Any]) -> List[Dict]:\n \"\"\"\n Score abstracts using the configured model type.\n\n Args:\n data (List[Dict]): List of dictionaries containing 'title', 'abstract', and 'gt_score'.\n\n Returns:\n List[Dict]: List of results with 'title', 'score', 'evaluation', 'abstract', and 'gt_score'.\n \"\"\"\n if isinstance(data[0], str):\n data = [{\"title\": \"\", \"gt_score\":0, \"abstract\": d} for d in data]\n\n if self.config.model_type == \"vllm\":\n return await self.score_with_vllm(data)\n elif self.config.model_type == \"api\":\n return await self.score_with_api(data)\n else:\n raise ValueError(\"Invalid model_type. Must be 'vllm' or 'api'.\")\n\nif __name__ == \"__main__\":\n async def test_reward_model():\n # Sample data for testing\n sample_data = [\n {\n \"title\": \"A Novel Approach to Quantum Computing\",\n \"abstract\": \"This proposal introduces a new quantum algorithm that enhances computational efficiency by leveraging entangled states in a scalable architecture. The approach is validated through simulations showing a 20% improvement over existing methods.\",\n \"gt_score\": 7.5\n },\n {\n \"title\": \"AI-Driven Climate Modeling\",\n \"abstract\": \"We propose an AI-based framework for improving climate predictions using deep learning to integrate heterogeneous environmental data. Preliminary results demonstrate enhanced accuracy in long-term forecasts.\",\n \"gt_score\": 8.0\n }\n ]\n\n # Test with vLLM model (commented out because it requires a local model and GPU)\n\n # try:\n # vllm_model = RewardModel(\n # model_type=\"vllm\",\n # model_name=\"/data/zhuotaodeng/yzj/alpha_research_model/qwen25_grm_iclr_boxed/checkpoint-180\",\n # jsonl_file=\"vllm_results.jsonl\"\n # )\n # vllm_results = await vllm_model.score_research_proposal(sample_data)\n # print(\"vLLM Results:\")\n # for result in vllm_results:\n # print(f\"Title: {result['title']}, Score: {result['score']}, Evaluation: {result['evaluation'][:50]}...\")\n # except Exception as e:\n # print(f\"vLLM test failed: {e}\")\n\n\n # Test with API model (requires valid API key and base URL)\n try:\n # Replace with your actual API key and base URL\n api_key = \"sk-2c3f1f58031b4b86afdb6a8192ea02e2\"\n base_url = \"https://api.deepseek.com\"\n\n config = RewardModelConfig(\n model_type=\"api\",\n model_name=\"deepseek-chat\",\n api_key=api_key,\n base_url=base_url,\n jsonl_file=\"api_results.jsonl\",\n max_retries=3,\n retry_delay=1\n ) \n \n api_model = RewardModel(config)\n api_results = await api_model.score_research_proposal(sample_data)\n print(\"API Results:\")\n for result in api_results:\n print(f\"Title: {result['title']}, Score: {result['score']}, Evaluation: {result['evaluation'][:50]}...\")\n except Exception as e:\n print(f\"API test failed: {e}\")\n\n # Run the async test\n asyncio.run(test_reward_model())\n"}}, "evolve_agent/prompt/__init__.py": {"76bbd8e5d7f00285af76350180ad9af8ca74b6b4": {"t": 1785599214.4152956, "n": 198, "h": "98db2c3b6cf11495", "c": "\"\"\"\nPrompt module initialization\n\"\"\"\n\nfrom evolve_agent.prompt.sampler import PromptSampler\nfrom evolve_agent.prompt.templates import TemplateManager\n\n__all__ = [\"PromptSampler\", \"TemplateManager\"]\n"}}, "evolve_agent/llm/openai.py": {"8c9dde0b3a4d9b6b5c2ca6669a485161f88258bd": {"t": 1785599214.4193788, "n": 4564, "h": "0bfc529021180eff", "c": "\"\"\"\nOpenAI API interface for LLMs\n\"\"\"\n\nimport asyncio\nimport logging\nimport time\nfrom typing import Any, Dict, List, Optional, Union\n\nimport openai\nfrom openai import AsyncOpenAI\nfrom openai import AsyncAzureOpenAI\n\nfrom evolve_agent.config import LLMConfig\nfrom evolve_agent.llm.base import LLMInterface\n\nlogger = logging.getLogger(__name__)\n\nclass OpenAILLM(LLMInterface):\n \"\"\"LLM interface using OpenAI-compatible APIs\"\"\"\n\n def __init__(\n self,\n model_cfg: Optional[dict] = None,\n ):\n self.model = model_cfg.name\n self.system_message = model_cfg.system_message\n self.temperature = model_cfg.temperature\n self.top_p = model_cfg.top_p\n self.max_tokens = model_cfg.max_tokens\n self.timeout = model_cfg.timeout\n self.retries = model_cfg.retries\n self.retry_delay = model_cfg.retry_delay\n self.api_base = model_cfg.api_base\n self.api_key = model_cfg.api_key\n\n # Set up async API client\n self.client = AsyncOpenAI(\n api_key=self.api_key,\n base_url=self.api_base,\n )\n # self.client = AsyncOpenAI(\n # api_key=self.api_key,\n # azure_endpoint=self.api_base,\n # api_version=\"2024-12-01-preview\",\n # )\n\n logger.info(f\"Initialized OpenAI LLM with model: {self.model}\")\n\n async def generate(self, prompt: str, **kwargs) -> str:\n \"\"\"Generate text from a prompt\"\"\"\n return await self.generate_with_context(\n system_message=self.system_message,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n **kwargs,\n )\n\n async def generate_with_context(\n self, system_message: str, messages: List[Dict[str, str]], **kwargs\n ) -> str:\n \"\"\"Generate text using a system message and conversational context\"\"\"\n # Prepare messages with system message\n formatted_messages = [{\"role\": \"system\", \"content\": system_message}]\n formatted_messages.extend(messages)\n\n # Set up generation parameters\n if self.api_base == \"https://api.openai.com/v1\" and str(self.model).lower().startswith(\"o\"):\n # For o-series models\n params = {\n \"model\": self.model,\n \"messages\": formatted_messages,\n \"max_completion_tokens\": kwargs.get(\"max_tokens\", self.max_tokens),\n }\n else:\n params = {\n \"model\": self.model,\n \"messages\": formatted_messages,\n \"temperature\": kwargs.get(\"temperature\", self.temperature),\n \"top_p\": kwargs.get(\"top_p\", self.top_p),\n \"max_tokens\": kwargs.get(\"max_tokens\", self.max_tokens),\n }\n\n # Attempt the API call with retries\n retries = kwargs.get(\"retries\", self.retries)\n retry_delay = kwargs.get(\"retry_delay\", self.retry_delay)\n timeout = kwargs.get(\"timeout\", self.timeout)\n\n for attempt in range(retries + 1):\n try:\n response = await asyncio.wait_for(self._call_api(params), timeout=timeout)\n return response\n except asyncio.TimeoutError:\n if attempt < retries:\n logger.warning(f\"Timeout on attempt {attempt + 1}/{retries + 1}. Retrying...\")\n await asyncio.sleep(retry_delay)\n else:\n logger.error(f\"All {retries + 1} attempts failed with timeout\")\n raise\n except Exception as e:\n if attempt < retries:\n logger.warning(\n f\"Error on attempt {attempt + 1}/{retries + 1}: {str(e)}. Retrying...\"\n )\n await asyncio.sleep(retry_delay)\n else:\n logger.error(f\"All {retries + 1} attempts failed with error: {str(e)}\")\n raise\n\n async def _call_api(self, params: Dict[str, Any]) -> str:\n \"\"\"Make the actual API call\"\"\"\n # Use native async API call\n response = await self.client.chat.completions.create(**params)\n # Logging of system prompt, user message and response content\n prompt = params[\"messages\"][0][\"content\"] + '\\n' + params[\"messages\"][1][\"content\"]\n logger.info('=' * 100)\n logger.info(f\"API parameters: {prompt}\")\n logger.info('=' * 100)\n logger.info(f\"API response: {response.choices[0].message.content}\")\n logger.info('=' * 100)\n return response.choices[0].message.content\n"}}, "evolve_agent/prompt/templates.py": {"81847fc8598b23294caf72dcba1b446cb2500c74": {"t": 1785599214.4688272, "n": 6555, "h": "da12fe7a0c648b72", "c": "\"\"\"\nPrompt templates for EvolveAgent\n\"\"\"\n\nimport os\nfrom pathlib import Path\nfrom typing import Dict, List, Optional, Union\n\n# Base system message template for evolution\nBASE_SYSTEM_TEMPLATE = \"\"\"You are an expert software developer tasked with iteratively improving a codebase.\nYour job is to analyze the current program and suggest improvements based on feedback from previous attempts.\nFocus on making targeted changes that will increase the program's performance metrics.\n\"\"\"\n\nBASE_EVALUATOR_SYSTEM_TEMPLATE = \"\"\"You are an expert code reviewer.\nYour job is to analyze the provided code and evaluate it systematically.\"\"\"\n\n# User message template for diff-based evolution\nDIFF_USER_TEMPLATE = \"\"\"# Current Program Information\n- Current performance metrics: {metrics}\n- Areas identified for improvement: {improvement_areas}\n\n{artifacts}\n\n# Program Evolution History\n{evolution_history}\n\n# Current Program\n```{language}\n{current_program}\n```\n\n# Task\nSuggest improvements to the program that will lead to better performance on the specified metrics.\n\nYou MUST use the exact SEARCH/REPLACE diff format shown below to indicate changes:\n\n<<<<<<< SEARCH\n# Original code to find and replace (must match exactly)\n=======\n# New replacement code\n>>>>>>> REPLACE\n\nExample of valid diff format:\n<<<<<<< SEARCH\nfor i in range(m):\n for j in range(p):\n for k in range(n):\n C[i, j] += A[i, k] * B[k, j]\n=======\n# Reorder loops for better memory access pattern\nfor i in range(m):\n for k in range(n):\n for j in range(p):\n C[i, j] += A[i, k] * B[k, j]\n>>>>>>> REPLACE\n\nYou can suggest multiple changes. Each SEARCH section must exactly match code in the current program.\nBe thoughtful about your changes and explain your reasoning thoroughly.\n\nIMPORTANT: Do not rewrite the entire program - focus on targeted improvements.\n\"\"\"\n\n\nDIFF_USER_TEMPLATE_PROPOSAL = \"\"\"# Previous Proposal: \n{parent_proposal_text}\n\n# Previous Program:\n```{language}\n{parent_program}\n```\n\n# Previous Performance Metrics: \n{metrics}\n\n# Areas Identified for Improvement: \n{improvement_areas}\n\n{artifacts}\n\n# Program Evolution History\n{evolution_history}\n\n# Current Proposal\n{current_proposal_text}\n\n# Task\nSuggest improvements to the program that will lead to better performance on the specified metrics.\n\nYou MUST use the exact SEARCH/REPLACE diff format shown below to indicate changes:\n\n<<<<<<< SEARCH\n# Original code to find and replace (must match exactly)\n=======\n# New replacement code\n>>>>>>> REPLACE\n\nExample of valid diff format:\n<<<<<<< SEARCH\nfor i in range(m):\n for j in range(p):\n for k in range(n):\n C[i, j] += A[i, k] * B[k, j]\n=======\n# Reorder loops for better memory access pattern\nfor i in range(m):\n for k in range(n):\n for j in range(p):\n C[i, j] += A[i, k] * B[k, j]\n>>>>>>> REPLACE\n\nYou can suggest multiple changes. Each SEARCH section must exactly match code in the current program.\nBe thoughtful about your changes and explain your reasoning thoroughly.\n\nIMPORTANT: Do not rewrite the entire program - focus on targeted improvements.\n\"\"\"\n\n\n\n# User message template for full rewrite\nFULL_REWRITE_USER_TEMPLATE = \"\"\"# Current Program Information\n- Current performance metrics: {metrics}\n- Areas identified for improvement: {improvement_areas}\n\n{artifacts}\n\n# Program Evolution History\n{evolution_history}\n\n# Current Program\n```{language}\n{current_program}\n```\n\n# Task\nRewrite the program to improve its performance on the specified metrics.\nProvide the complete new program code.\n\nIMPORTANT: Make sure your rewritten program maintains the same inputs and outputs\nas the original program, but with improved internal implementation.\n\n```{language}\n# Your rewritten program here\n```\n\"\"\"\n\n# Template for formatting evolution history\nEVOLUTION_HISTORY_TEMPLATE = \"\"\"## Previous Attempts\n\n{previous_attempts}\n\n## Top Performing Programs\n\n{top_programs}\n\"\"\"\n\n# Template for formatting a previous attempt\nPREVIOUS_ATTEMPT_TEMPLATE = \"\"\"### Attempt {attempt_number}\n- Changes: {changes}\n- Performance: {performance}\n- Outcome: {outcome}\n\"\"\"\n\n# Template for formatting a top program\nTOP_PROGRAM_TEMPLATE = \"\"\"### Program {program_number} (Score: {score})\n```{language}\n{program_snippet}\n```\nKey features: {key_features}\n\"\"\"\n\n# Template for evaluating a program via an LLM\nEVALUATION_TEMPLATE = \"\"\"Evaluate the following code on a scale of 0.0 to 1.0 for the following metrics:\n1. Readability: How easy is the code to read and understand?\n2. Maintainability: How easy would the code be to maintain and modify?\n3. Efficiency: How efficient is the code in terms of time and space complexity?\n\nFor each metric, provide a score between 0.0 and 1.0, where 1.0 is best.\n\nCode to evaluate:\n```python\n{current_program}\n```\n\nReturn your evaluation as a JSON object with the following format:\n{{\n \"readability\": [score],\n \"maintainability\": [score],\n \"efficiency\": [score],\n \"reasoning\": \"[brief explanation of scores]\"\n}}\n\"\"\"\n\n\n# Default templates dictionary\nDEFAULT_TEMPLATES = {\n \"system_message\": BASE_SYSTEM_TEMPLATE,\n \"evaluator_system_message\": BASE_EVALUATOR_SYSTEM_TEMPLATE,\n # \"diff_user\": DIFF_USER_TEMPLATE,\n \"full_rewrite_user\": FULL_REWRITE_USER_TEMPLATE,\n \"evolution_history\": EVOLUTION_HISTORY_TEMPLATE,\n \"previous_attempt\": PREVIOUS_ATTEMPT_TEMPLATE,\n \"top_program\": TOP_PROGRAM_TEMPLATE,\n \"evaluation\": EVALUATION_TEMPLATE,\n \"diff_user\": DIFF_USER_TEMPLATE_PROPOSAL\n}\n\n\nclass TemplateManager:\n \"\"\"Manages templates for prompt generation\"\"\"\n\n def __init__(self, template_dir: Optional[str] = None):\n self.templates = DEFAULT_TEMPLATES.copy()\n\n # Load templates from directory if provided\n if template_dir and os.path.isdir(template_dir):\n self._load_templates_from_dir(template_dir)\n\n def _load_templates_from_dir(self, template_dir: str) -> None:\n \"\"\"Load templates from a directory\"\"\"\n for file_path in Path(template_dir).glob(\"*.txt\"):\n template_name = file_path.stem\n with open(file_path, \"r\") as f:\n self.templates[template_name] = f.read()\n\n def get_template(self, template_name: str) -> str:\n \"\"\"Get a template by name\"\"\"\n if template_name not in self.templates:\n raise ValueError(f\"Template '{template_name}' not found\")\n return self.templates[template_name]\n\n def add_template(self, template_name: str, template: str) -> None:\n \"\"\"Add or update a template\"\"\"\n self.templates[template_name] = template\n"}}, "configs/island_examples.yaml": {"40c2c696646bd4a6efc6e8fb247a9544af1b5ad4": {"t": 1785599214.490425, "n": 3066, "h": "e4653ec7e32abe4c", "c": "# EvolveAgent Island-Based Evolution Configuration Examples\n# Different configurations for various use cases\n\n# Configuration for Maximum Diversity (Many Islands, Frequent Migration)\n# Use this when you want to explore the search space thoroughly\n# Good for: Complex problems, avoiding local optima, long runs\nmax_diversity:\n database:\n num_islands: 10 # More islands = more diversity\n migration_interval: 25 # More frequent migration\n migration_rate: 0.2 # Higher migration rate\n population_size: 1000\n archive_size: 200\n\n# Configuration for Focused Exploration (Few Islands, Rare Migration) \n# Use this when you want deeper exploration within each island\n# Good for: Problems with clear structure, shorter runs\nfocused_exploration:\n database:\n num_islands: 3 # Fewer islands = deeper exploration\n migration_interval: 100 # Less frequent migration\n migration_rate: 0.05 # Lower migration rate\n population_size: 500\n archive_size: 50\n\n# Configuration for Balanced Approach (Default Settings)\n# Use this as a starting point for most problems\n# Good for: General use, medium-length runs\nbalanced:\n database:\n num_islands: 5 # Balanced number of islands\n migration_interval: 50 # Moderate migration frequency\n migration_rate: 0.1 # Moderate migration rate\n population_size: 1000\n archive_size: 100\n\n# Configuration for Quick Exploration (Small Scale)\n# Use this for rapid prototyping and testing\n# Good for: Small problems, quick experiments\nquick_exploration:\n database:\n num_islands: 3\n migration_interval: 20\n migration_rate: 0.15\n population_size: 200\n archive_size: 30\n\n# Configuration for Large-Scale Evolution (High Performance)\n# Use this for complex problems requiring extensive search\n# Good for: Complex optimization, long evolutionary runs\nlarge_scale:\n database:\n num_islands: 15 # Many islands for parallel exploration\n migration_interval: 75 # Balanced migration timing\n migration_rate: 0.08 # Conservative migration rate\n population_size: 2000 # Large populations\n archive_size: 300\n\n# Guidelines for choosing parameters:\n#\n# num_islands:\n# - More islands = more diversity, slower convergence\n# - Fewer islands = faster convergence, risk of premature convergence\n# - Recommended: 3-10 for most problems\n#\n# migration_interval:\n# - Lower values = more frequent knowledge sharing\n# - Higher values = more independent evolution\n# - Recommended: 25-100 generations\n#\n# migration_rate:\n# - Higher values = faster knowledge propagation\n# - Lower values = preserve island diversity longer\n# - Recommended: 0.05-0.2 (5%-20%)\n#\n# Rule of thumb:\n# - Complex problems → More islands, less frequent migration\n# - Simple problems → Fewer islands, more frequent migration\n# - Long runs → More islands to maintain diversity\n# - Short runs → Fewer islands for faster convergence\n"}}, "evolve_agent/utils/format_utils.py": {"4dd83c7ac0e4e6d94008997b0b372e9bf455a6b4": {"t": 1785599214.7944198, "n": 2119, "h": "645370ca9688af73", "c": "\"\"\"\nUtility functions for formatting output\n\"\"\"\n\nfrom typing import Any, Dict\n\n\ndef format_metrics_safe(metrics: Dict[str, Any]) -> str:\n \"\"\"\n Safely format metrics dictionary for logging, handling both numeric and string values.\n\n Args:\n metrics: Dictionary of metric names to values\n\n Returns:\n Formatted string representation of metrics\n \"\"\"\n if not metrics:\n return \"\"\n\n formatted_parts = []\n for name, value in metrics.items():\n # Check if value is numeric (int, float)\n if isinstance(value, (int, float)):\n try:\n # Only apply float formatting to numeric values\n formatted_parts.append(f\"{name}={value:.4f}\")\n except (ValueError, TypeError):\n # Fallback to string representation if formatting fails\n formatted_parts.append(f\"{name}={value}\")\n else:\n # For non-numeric values (strings, etc.), just convert to string\n formatted_parts.append(f\"{name}={value}\")\n\n return \", \".join(formatted_parts)\n\n\ndef format_improvement_safe(parent_metrics: Dict[str, Any], child_metrics: Dict[str, Any]) -> str:\n \"\"\"\n Safely format improvement metrics for logging.\n\n Args:\n parent_metrics: Parent program metrics\n child_metrics: Child program metrics\n\n Returns:\n Formatted string representation of improvements\n \"\"\"\n if not parent_metrics or not child_metrics:\n return \"\"\n\n improvement_parts = []\n for metric, child_value in child_metrics.items():\n if metric in parent_metrics:\n parent_value = parent_metrics[metric]\n # Only calculate improvement for numeric values\n if isinstance(child_value, (int, float)) and isinstance(parent_value, (int, float)):\n try:\n diff = child_value - parent_value\n improvement_parts.append(f\"{metric}={diff:+.4f}\")\n except (ValueError, TypeError):\n # Skip non-numeric comparisons\n continue\n\n return \", \".join(improvement_parts)\n"}}, "evolve_agent/utils/code_utils.py": {"60fb63001cfc55154adb95ca7876d7cb0f40e132": {"t": 1785599214.7954454, "n": 5959, "h": "f058397078e17a09", "c": "\"\"\"\nUtilities for code parsing, diffing, and manipulation\n\"\"\"\n\nimport re\nfrom typing import Dict, List, Optional, Tuple, Union\n\n\ndef parse_evolve_blocks(code: str) -> List[Tuple[int, int, str]]:\n \"\"\"\n Parse evolve blocks from code\n\n Args:\n code: Source code with evolve blocks\n\n Returns:\n List of tuples (start_line, end_line, block_content)\n \"\"\"\n lines = code.split(\"\\n\")\n blocks = []\n\n in_block = False\n start_line = -1\n block_content = []\n\n for i, line in enumerate(lines):\n if \"# EVOLVE-BLOCK-START\" in line:\n in_block = True\n start_line = i\n block_content = []\n elif \"# EVOLVE-BLOCK-END\" in line and in_block:\n in_block = False\n blocks.append((start_line, i, \"\\n\".join(block_content)))\n elif in_block:\n block_content.append(line)\n\n return blocks\n\n\ndef apply_diff(original_code: str, diff_text: str) -> str:\n \"\"\"\n Apply a diff to the original code\n\n Args:\n original_code: Original source code\n diff_text: Diff in the SEARCH/REPLACE format\n\n Returns:\n Modified code\n \"\"\"\n # Split into lines for easier processing\n original_lines = original_code.split(\"\\n\")\n result_lines = original_lines.copy()\n\n # Extract diff blocks\n diff_blocks = extract_diffs(diff_text)\n\n # Apply each diff block\n for search_text, replace_text in diff_blocks:\n search_lines = search_text.split(\"\\n\")\n replace_lines = replace_text.split(\"\\n\")\n\n # Find where the search pattern starts in the original code\n for i in range(len(result_lines) - len(search_lines) + 1):\n if result_lines[i : i + len(search_lines)] == search_lines:\n # Replace the matched section\n result_lines[i : i + len(search_lines)] = replace_lines\n break\n\n return \"\\n\".join(result_lines)\n\n\ndef extract_diffs(diff_text: str) -> List[Tuple[str, str]]:\n \"\"\"\n Extract diff blocks from the diff text\n\n Args:\n diff_text: Diff in the SEARCH/REPLACE format\n\n Returns:\n List of tuples (search_text, replace_text)\n \"\"\"\n diff_pattern = r\"<<<<<<< SEARCH\\n(.*?)=======\\n(.*?)>>>>>>> REPLACE\"\n diff_blocks = re.findall(diff_pattern, diff_text, re.DOTALL)\n return [(match[0].rstrip(), match[1].rstrip()) for match in diff_blocks]\n\n\ndef parse_full_rewrite(llm_response: str, language: str = \"python\") -> Optional[str]:\n \"\"\"\n Extract a full rewrite from an LLM response\n\n Args:\n llm_response: Response from the LLM\n language: Programming language\n\n Returns:\n Extracted code or None if not found\n \"\"\"\n code_block_pattern = r\"```\" + language + r\"\\n(.*?)```\"\n matches = re.findall(code_block_pattern, llm_response, re.DOTALL)\n\n if matches:\n return matches[0].strip()\n\n # Fallback to any code block\n code_block_pattern = r\"```(.*?)```\"\n matches = re.findall(code_block_pattern, llm_response, re.DOTALL)\n\n if matches:\n return matches[0].strip()\n\n # Fallback to plain text\n return llm_response\n\n\ndef format_diff_summary(diff_blocks: List[Tuple[str, str]]) -> str:\n \"\"\"\n Create a human-readable summary of the diff\n\n Args:\n diff_blocks: List of (search_text, replace_text) tuples\n\n Returns:\n Summary string\n \"\"\"\n summary = []\n\n for i, (search_text, replace_text) in enumerate(diff_blocks):\n search_lines = search_text.strip().split(\"\\n\")\n replace_lines = replace_text.strip().split(\"\\n\")\n\n # Create a short summary\n if len(search_lines) == 1 and len(replace_lines) == 1:\n summary.append(f\"Change {i+1}: '{search_lines[0]}' to '{replace_lines[0]}'\")\n else:\n search_summary = (\n f\"{len(search_lines)} lines\" if len(search_lines) > 1 else search_lines[0]\n )\n replace_summary = (\n f\"{len(replace_lines)} lines\" if len(replace_lines) > 1 else replace_lines[0]\n )\n summary.append(f\"Change {i+1}: Replace {search_summary} with {replace_summary}\")\n\n return \"\\n\".join(summary)\n\n\ndef calculate_edit_distance(code1: str, code2: str) -> int:\n \"\"\"\n Calculate the Levenshtein edit distance between two code snippets\n\n Args:\n code1: First code snippet\n code2: Second code snippet\n\n Returns:\n Edit distance (number of operations needed to transform code1 into code2)\n \"\"\"\n if code1 == code2:\n return 0\n\n # Simple implementation of Levenshtein distance\n m, n = len(code1), len(code2)\n dp = [[0 for _ in range(n + 1)] for _ in range(m + 1)]\n\n for i in range(m + 1):\n dp[i][0] = i\n\n for j in range(n + 1):\n dp[0][j] = j\n\n for i in range(1, m + 1):\n for j in range(1, n + 1):\n cost = 0 if code1[i - 1] == code2[j - 1] else 1\n dp[i][j] = min(\n dp[i - 1][j] + 1, # deletion\n dp[i][j - 1] + 1, # insertion\n dp[i - 1][j - 1] + cost, # substitution\n )\n\n return dp[m][n]\n\n\ndef extract_code_language(code: str) -> str:\n \"\"\"\n Try to determine the language of a code snippet\n\n Args:\n code: Code snippet\n\n Returns:\n Detected language or \"unknown\"\n \"\"\"\n # Look for common language signatures\n if re.search(r\"^(import|from|def|class)\\s\", code, re.MULTILINE):\n return \"python\"\n elif re.search(r\"^(package|import java|public class)\", code, re.MULTILINE):\n return \"java\"\n elif re.search(r\"^(#include|int main|void main)\", code, re.MULTILINE):\n return \"cpp\"\n elif re.search(r\"^(function|var|let|const|console\\.log)\", code, re.MULTILINE):\n return \"javascript\"\n elif re.search(r\"^(module|fn|let mut|impl)\", code, re.MULTILINE):\n return \"rust\"\n elif re.search(r\"^(SELECT|CREATE TABLE|INSERT INTO)\", code, re.MULTILINE):\n return \"sql\"\n\n return \"unknown\"\n"}}, "compute.py": {"a660f5e31081ad2a884742c4c4834b0a2ef63d87": {"t": 1785599214.8260803, "n": 2687, "h": "8a7547265e1eedfc", "c": "import math\n\nbaselines = {\n \"packing_circles_26\": {\"s_baseline\": 2.634, \"higher_better\": 1},\n \"packind_circles_32\": {\"s_baseline\": 2.936, \"higher_better\": 1},\n \"minizing_raio_max_min_distance_d2_n16\": {\"s_baseline\": 12.89, \"higher_better\": -1},\n \"minizing_raio_max_min_distance_d3_n14\": {\"s_baseline\": 4.168, \"higher_better\": -1},\n \"third_autocorrelation_inequality\": {\"s_baseline\": 1.4581, \"higher_better\": -1},\n # Added benchmarks (larger-is-better where applicable)\n \"kissing_number_d11\": {\"s_baseline\": 592.0, \"higher_better\": 1},\n \"spherical_code_d3_n30\": {\"s_baseline\": 0.6736467551690225, \"higher_better\": 1},\n \"heilbronn_in_the_unit_square_n16\": {\"s_baseline\": 7.0/341.0, \"higher_better\": 1},\n \"littlewood_polynomials_n512\": {\"s_baseline\": 0.04105, \"higher_better\": 1},\n #\"riesz_energy_n20_s1\": {\"s_baseline\": 0.001013, \"higher_better\": 1},\n \"MSTD_n30\": {\"s_baseline\": 1.04, \"higher_better\": 1},\n \"autoconvolution_peak_minimization\": {\"s_baseline\": 0.6667, \"higher_better\": 1}\n}\n\nresults = {\n \"packing_circles_26\": {\"s_best\": 2.6359829561164743, \"round\": 40657},\n \"packind_circles_32\": {\"s_best\": 2.939520304932057, \"round\": 40657},\n \"minizing_raio_max_min_distance_d2_n16\": {\"s_best\": 12.92, \"round\": 5000},\n \"minizing_raio_max_min_distance_d3_n14\": {\"s_best\": 5.198, \"round\": 5000},\n \"third_autocorrelation_inequality\": {\"s_best\": 0, \"round\": 5000},\n # Placeholders for newly added benchmarks (0 indicates not yet attempted)\n \"kissing_number_d11\": {\"s_best\": 502.0, \"round\": 5000},\n \"spherical_code_d3_n30\": {\"s_best\": 0.6381359964781541, \"round\": 5000},\n \"heilbronn_in_the_unit_square_n16\": {\"s_best\": 0, \"round\": 5000},\n \"littlewood_polynomials_n512\": {\"s_best\": 0, \"round\": 5000},\n #\"riesz_energy_n20_s1\": {\"s_best\": 0, \"round\": 5000},\n \"MSTD_n30\": {\"s_best\": 0, \"round\": 5000},\n \"autoconvolution_peak_minimization\": {\"s_best\": 0, \"round\": 5000}\n}\n\ndef compute_excel_best(results):\n problems = list(baselines.keys())\n num_problems = len(problems)\n total = 0.0\n for problem in problems:\n s_baseline = baselines[problem]['s_baseline']\n higher_better = baselines[problem]['higher_better']\n s_best = results[problem]['s_best']\n n_round = results[problem]['round']\n if s_best == 0:\n s_excess = 0 # Assuming s_best == 0 indicates failure/no improvement\n else:\n improvement = (s_best - s_baseline) * higher_better\n s_excess = max(improvement, 0)\n contrib = s_excess / (n_round / 1000000)\n total += contrib\n excel_best = total / num_problems\n return excel_best\n\nprint(compute_excel_best(results))"}}, "evolve_agent/utils/async_utils.py": {"22653a5e1c493199f5b3a51cc6969c0d6fa712eb": {"t": 1785599214.8497045, "n": 6809, "h": "f18985f79273df87", "c": "\"\"\"\nAsync utilities for EvolveAgent\n\"\"\"\n\nimport asyncio\nimport functools\nimport logging\nimport time\nfrom typing import Any, Callable, Dict, List, Optional, TypeVar, Union\n\nlogger = logging.getLogger(__name__)\n\nT = TypeVar(\"T\")\n\n\ndef run_in_executor(f: Callable) -> Callable:\n \"\"\"\n Decorator to run a synchronous function in an executor\n\n Args:\n f: Function to decorate\n\n Returns:\n Decorated function that runs in an executor\n \"\"\"\n\n @functools.wraps(f)\n async def wrapper(*args: Any, **kwargs: Any) -> Any:\n loop = asyncio.get_event_loop()\n return await loop.run_in_executor(None, functools.partial(f, *args, **kwargs))\n\n return wrapper\n\n\nasync def run_with_timeout(\n coro: Callable, timeout: float, *args: Any, timeout_error_value: Any = None, **kwargs: Any\n) -> Any:\n \"\"\"\n Run a coroutine with a timeout, returning a default value on timeout\n\n Args:\n coro: Coroutine function to run\n timeout: Timeout in seconds\n *args: Arguments to pass to the coroutine\n timeout_error_value: Value to return on timeout (default: {\"error\": 0.0, \"timeout\": True})\n **kwargs: Keyword arguments to pass to the coroutine\n\n Returns:\n Result of the coroutine or timeout_error_value on timeout\n \"\"\"\n if timeout_error_value is None:\n timeout_error_value = {\"error\": 0.0, \"timeout\": True}\n\n try:\n return await asyncio.wait_for(coro(*args, **kwargs), timeout=timeout)\n except asyncio.TimeoutError:\n logger.warning(f\"Operation timed out after {timeout}s\")\n return timeout_error_value\n\n\nasync def run_sync_with_timeout(\n func: Callable, timeout: float, *args: Any, timeout_error_value: Any = None, **kwargs: Any\n) -> Any:\n \"\"\"\n Run a synchronous function in an executor with a timeout\n\n Args:\n func: Synchronous function to run\n timeout: Timeout in seconds\n *args: Arguments to pass to the function\n timeout_error_value: Value to return on timeout (default: {\"error\": 0.0, \"timeout\": True})\n **kwargs: Keyword arguments to pass to the function\n\n Returns:\n Result of the function or timeout_error_value on timeout\n \"\"\"\n if timeout_error_value is None:\n timeout_error_value = {\"error\": 0.0, \"timeout\": True}\n\n try:\n loop = asyncio.get_event_loop()\n task = loop.run_in_executor(None, functools.partial(func, *args, **kwargs))\n return await asyncio.wait_for(task, timeout=timeout)\n except asyncio.TimeoutError:\n logger.warning(f\"Sync operation timed out after {timeout}s\")\n return timeout_error_value\n\n\nasync def gather_with_concurrency(\n n: int, *tasks: asyncio.Future, return_exceptions: bool = False\n) -> List[Any]:\n \"\"\"\n Run tasks with a concurrency limit\n\n Args:\n n: Maximum number of tasks to run concurrently\n *tasks: Tasks to run\n return_exceptions: Whether to return exceptions instead of raising them\n\n Returns:\n List of task results\n \"\"\"\n semaphore = asyncio.Semaphore(n)\n\n async def sem_task(task: asyncio.Future) -> Any:\n async with semaphore:\n return await task\n\n return await asyncio.gather(\n *(sem_task(task) for task in tasks), return_exceptions=return_exceptions\n )\n\n\nasync def retry_async(\n coro: Callable,\n *args: Any,\n retries: int = 3,\n delay: float = 1.0,\n backoff: float = 2.0,\n exceptions: Union[Exception, tuple] = Exception,\n **kwargs: Any,\n) -> Any:\n \"\"\"\n Retry an async function with exponential backoff\n\n Args:\n coro: Coroutine function to retry\n *args: Arguments to pass to the coroutine\n retries: Maximum number of retries\n delay: Initial delay between retries (seconds)\n backoff: Multiplier for delay between retries\n exceptions: Exception(s) to catch\n **kwargs: Keyword arguments to pass to the coroutine\n\n Returns:\n Result of the coroutine\n\n Raises:\n The last exception caught if all retries fail\n \"\"\"\n last_exception = None\n current_delay = delay\n\n for i in range(retries + 1):\n try:\n return await coro(*args, **kwargs)\n except exceptions as e:\n last_exception = e\n if i < retries:\n logger.warning(\n f\"Retry {i+1}/{retries} failed with {type(e).__name__}: {str(e)}. \"\n f\"Retrying in {current_delay:.2f}s...\"\n )\n await asyncio.sleep(current_delay)\n current_delay *= backoff\n else:\n logger.error(\n f\"All {retries+1} attempts failed. Last error: {type(e).__name__}: {str(e)}\"\n )\n\n if last_exception:\n raise last_exception\n\n return None # Should never reach here\n\n\nclass TaskPool:\n \"\"\"\n A simple task pool for managing and limiting concurrent tasks\n \"\"\"\n\n def __init__(self, max_concurrency: int = 10):\n self.max_concurrency = max_concurrency\n self._semaphore: Optional[asyncio.Semaphore] = None\n self.tasks: List[asyncio.Task] = []\n\n @property\n def semaphore(self) -> asyncio.Semaphore:\n \"\"\"Lazy-initialize the semaphore when first needed\"\"\"\n if self._semaphore is None:\n self._semaphore = asyncio.Semaphore(self.max_concurrency)\n return self._semaphore\n\n async def run(self, coro: Callable, *args: Any, **kwargs: Any) -> Any:\n \"\"\"\n Run a coroutine in the pool\n\n Args:\n coro: Coroutine function to run\n *args: Arguments to pass to the coroutine\n **kwargs: Keyword arguments to pass to the coroutine\n\n Returns:\n Result of the coroutine\n \"\"\"\n async with self.semaphore:\n return await coro(*args, **kwargs)\n\n def create_task(self, coro: Callable, *args: Any, **kwargs: Any) -> asyncio.Task:\n \"\"\"\n Create and track a task in the pool\n\n Args:\n coro: Coroutine function to run\n *args: Arguments to pass to the coroutine\n **kwargs: Keyword arguments to pass to the coroutine\n\n Returns:\n Task object\n \"\"\"\n task = asyncio.create_task(self.run(coro, *args, **kwargs))\n self.tasks.append(task)\n task.add_done_callback(lambda t: self.tasks.remove(t))\n return task\n\n async def wait_all(self) -> None:\n \"\"\"Wait for all tasks in the pool to complete\"\"\"\n if self.tasks:\n await asyncio.gather(*self.tasks)\n\n async def cancel_all(self) -> None:\n \"\"\"Cancel all tasks in the pool\"\"\"\n for task in self.tasks:\n task.cancel()\n\n if self.tasks:\n await asyncio.gather(*self.tasks, return_exceptions=True)\n"}}, "evolve_agent/utils/__init__.py": {"37bbd5f561ea0aca2e8be36174fc3c5884d7ea80": {"t": 1785599215.4177973, "n": 972, "h": "71446f77a829f673", "c": "\"\"\"\nUtilities module initialization\n\"\"\"\n\nfrom evolve_agent.utils.async_utils import (\n TaskPool,\n gather_with_concurrency,\n retry_async,\n run_in_executor,\n)\nfrom evolve_agent.utils.code_utils import (\n apply_diff,\n calculate_edit_distance,\n extract_code_language,\n extract_diffs,\n format_diff_summary,\n parse_evolve_blocks,\n parse_full_rewrite,\n)\nfrom evolve_agent.utils.format_utils import (\n format_metrics_safe,\n format_improvement_safe,\n)\nfrom evolve_agent.utils.metrics_utils import (\n safe_numeric_average,\n safe_numeric_sum,\n)\n\n__all__ = [\n \"TaskPool\",\n \"gather_with_concurrency\",\n \"retry_async\",\n \"run_in_executor\",\n \"apply_diff\",\n \"calculate_edit_distance\",\n \"extract_code_language\",\n \"extract_diffs\",\n \"format_diff_summary\",\n \"parse_evolve_blocks\",\n \"parse_full_rewrite\",\n \"format_metrics_safe\",\n \"format_improvement_safe\",\n \"safe_numeric_average\",\n \"safe_numeric_sum\",\n]\n"}}, "evolve_agent/llm/ensemble.py": {"528e3a8cca363e313691d6cf4cd3135bd30ead86": {"t": 1785599215.5478184, "n": 2723, "h": "6344bf658a26e0a0", "c": "\"\"\"\nModel ensemble for LLMs\n\"\"\"\n\nimport asyncio\nimport logging\nimport random\nfrom typing import Dict, List, Optional, Tuple\n\nfrom evolve_agent.llm.base import LLMInterface\nfrom evolve_agent.llm.openai import OpenAILLM\nfrom evolve_agent.config import LLMModelConfig\n\nlogger = logging.getLogger(__name__)\n\n\nclass LLMEnsemble:\n \"\"\"Ensemble of LLMs\"\"\"\n\n def __init__(self, models_cfg: List[LLMModelConfig]):\n self.models_cfg = models_cfg\n\n # Initialize models from the configuration\n self.models = [OpenAILLM(model_cfg) for model_cfg in models_cfg]\n\n # Extract and normalize model weights\n self.weights = [model.weight for model in models_cfg]\n total = sum(self.weights)\n self.weights = [w / total for w in self.weights]\n\n logger.info(\n f\"Initialized LLM ensemble with models: \"\n + \", \".join(\n f\"{model.name} (weight: {weight:.2f})\"\n for model, weight in zip(models_cfg, self.weights)\n )\n )\n\n async def generate(self, prompt: str, **kwargs) -> str:\n \"\"\"Generate text using a randomly selected model based on weights\"\"\"\n model = self._sample_model()\n return await model.generate(prompt, **kwargs)\n\n async def generate_with_context(\n self, system_message: str, messages: List[Dict[str, str]], **kwargs\n ) -> str:\n \"\"\"Generate text using a system message and conversational context\"\"\"\n model = self._sample_model()\n return await model.generate_with_context(system_message, messages, **kwargs)\n\n def _sample_model(self) -> LLMInterface:\n \"\"\"Sample a model from the ensemble based on weights\"\"\"\n index = random.choices(range(len(self.models)), weights=self.weights, k=1)[0]\n return self.models[index]\n\n async def generate_multiple(self, prompt: str, n: int, **kwargs) -> List[str]:\n \"\"\"Generate multiple texts in parallel\"\"\"\n tasks = [self.generate(prompt, **kwargs) for _ in range(n)]\n return await asyncio.gather(*tasks)\n\n async def parallel_generate(self, prompts: List[str], **kwargs) -> List[str]:\n \"\"\"Generate responses for multiple prompts in parallel\"\"\"\n tasks = [self.generate(prompt, **kwargs) for prompt in prompts]\n return await asyncio.gather(*tasks)\n\n async def generate_all_with_context(\n self, system_message: str, messages: List[Dict[str, str]], **kwargs\n ) -> str:\n \"\"\"Generate text using a all available models and average their returned metrics\"\"\"\n responses = []\n for model in self.models:\n responses.append(await model.generate_with_context(system_message, messages, **kwargs))\n return responses\n"}}, "benchmark/third_autocorrelation_inequality/initial_program.py": {"d122fc0e1dc100c3b2d554c12cbd6266ea60dfea": {"t": 1785599215.606694, "n": 3337, "h": "363eaa380aaf89e0", "c": "import numpy as np\nimport scipy.integrate\n\ndef calculate_c3_upper_bound(height_sequence):\n\n N = len(height_sequence)\n delta_x = 1 / (2 * N)\n\n def f(x):\n if -0.25 <= x <= 0.25:\n index = int((x - (-0.25)) / delta_x)\n if index == N:\n index -= 1\n return height_sequence[index]\n else:\n return 0.0\n\n integral_f = np.sum(height_sequence) * delta_x\n integral_sq = integral_f**2\n\n if integral_sq < 1e-18:\n return 0.0\n\n t_points = np.linspace(-0.5, 0.5, 2 * N + 1)\n \n max_conv_val = 0.0\n for t_val in t_points:\n\n lower_bound = max(-0.25, t_val - 0.25)\n upper_bound = min(0.25, t_val + 0.25)\n\n if upper_bound <= lower_bound:\n convolution_val = 0.0\n else:\n def integrand(x):\n return f(x) * f(t_val - x)\n \n convolution_val, _ = scipy.integrate.quad(integrand, lower_bound, upper_bound, limit=100)\n \n if abs(convolution_val) > max_conv_val:\n max_conv_val = abs(convolution_val)\n\n return max_conv_val / integral_sq\n\ndef genetic_algorithm(population_size, num_intervals, generations, mutation_rate, crossover_rate):\n\n population = np.random.rand(population_size, num_intervals) * 2 - 1\n\n best_solution = None\n best_fitness = 0.0\n\n for gen in range(generations):\n\n fitness_scores = np.array([calculate_c3_upper_bound(individual) for individual in population])\n\n current_best_idx = np.argmax(fitness_scores)\n if fitness_scores[current_best_idx] > best_fitness:\n best_fitness = fitness_scores[current_best_idx]\n best_solution = population[current_best_idx].copy()\n # print(f\"Generation {gen}: New best fitness = {best_fitness}\")\n\n\n new_population = np.zeros_like(population)\n for i in range(population_size):\n\n competitors_indices = np.random.choice(population_size, 2, replace=False)\n winner_idx = competitors_indices[np.argmax(fitness_scores[competitors_indices])]\n new_population[i] = population[winner_idx].copy()\n \n for i in range(0, population_size, 2):\n if np.random.rand() < crossover_rate:\n parent1 = new_population[i]\n parent2 = new_population[i+1]\n crossover_point = np.random.randint(1, num_intervals - 1)\n new_population[i] = np.concatenate((parent1[:crossover_point], parent2[crossover_point:]))\n new_population[i+1] = np.concatenate((parent2[:crossover_point], parent1[crossover_point:]))\n\n for i in range(population_size):\n if np.random.rand() < mutation_rate:\n mutation_point = np.random.randint(num_intervals)\n new_population[i, mutation_point] += np.random.normal(0, 0.1) \n\n new_population[i, mutation_point] = np.clip(new_population[i, mutation_point], -2, 2)\n\n population = new_population\n \n return best_solution\n\ndef find_better_c3_upper_bound():\n\n NUM_INTERVALS = 4\n POPULATION_SIZE = 2\n GENERATIONS = 10\n MUTATION_RATE = 0.1\n CROSSOVER_RATE = 0.8\n\n height_sequence_3 = genetic_algorithm(POPULATION_SIZE, NUM_INTERVALS, GENERATIONS, MUTATION_RATE, CROSSOVER_RATE)\n \n return height_sequence_3"}}, "reward_model/run.sh": {"220f3ded07b43a175c1d51f2641a16836478b2cc": {"t": 1785599215.6160278, "n": 92, "h": "4b1a57ea5dfd52fb", "c": "export CUDA_VISIBLE_DEVICES=0\npython /data/zhuotaodeng/yzj/alpha-research/idea-eval/grade.py"}}, "reward_model/train/download.py": {"b1cceb2f7bceba83d58af1e6a3b104661a7a4f99": {"t": 1785599215.630116, "n": 140, "h": "8b109f3f1934b123", "c": "from transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained('Qwen/Qwen2.5-Coder-32B-Instruct')\n\nprint(model)"}}, "evolve_agent/utils/metrics_utils.py": {"f2da61f49f0fe239290701bedbfdb12d89618e8f": {"t": 1785599215.6479092, "n": 1977, "h": "e12d6853cecbe3d8", "c": "\"\"\"\nSafe calculation utilities for metrics containing mixed types\n\"\"\"\n\nfrom typing import Any, Dict\n\n\ndef safe_numeric_average(metrics: Dict[str, Any]) -> float:\n \"\"\"\n Calculate the average of numeric values in a metrics dictionary,\n safely ignoring non-numeric values like strings.\n\n Args:\n metrics: Dictionary of metric names to values\n\n Returns:\n Average of numeric values, or 0.0 if no numeric values found\n \"\"\"\n if not metrics:\n return 0.0\n\n numeric_values = []\n for value in metrics.values():\n if isinstance(value, (int, float)):\n try:\n # Convert to float and check if it's a valid number\n float_val = float(value)\n if not (float_val != float_val): # Check for NaN (NaN != NaN is True)\n numeric_values.append(float_val)\n except (ValueError, TypeError, OverflowError):\n # Skip invalid numeric values\n continue\n\n if not numeric_values:\n return 0.0\n\n return sum(numeric_values) / len(numeric_values)\n\n\ndef safe_numeric_sum(metrics: Dict[str, Any]) -> float:\n \"\"\"\n Calculate the sum of numeric values in a metrics dictionary,\n safely ignoring non-numeric values like strings.\n\n Args:\n metrics: Dictionary of metric names to values\n\n Returns:\n Sum of numeric values, or 0.0 if no numeric values found\n \"\"\"\n if not metrics:\n return 0.0\n\n numeric_sum = 0.0\n for value in metrics.values():\n if isinstance(value, (int, float)):\n try:\n # Convert to float and check if it's a valid number\n float_val = float(value)\n if not (float_val != float_val): # Check for NaN (NaN != NaN is True)\n numeric_sum += float_val\n except (ValueError, TypeError, OverflowError):\n # Skip invalid numeric values\n continue\n\n return numeric_sum\n"}}, "evolve_agent/prompt/sampler.py": {"72a516f61401824c4e5ce4e5b80f4b4ed703c0a8": {"t": 1785599215.9085748, "n": 21545, "h": "cd55b9c870e809b2", "c": "\"\"\"\nPrompt sampling for EvolveAgent\n\"\"\"\n\nimport logging\nimport random\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nfrom evolve_agent.config import PromptConfig\nfrom evolve_agent.prompt.templates import TemplateManager\nfrom evolve_agent.utils.format_utils import format_metrics_safe\nfrom evolve_agent.utils.metrics_utils import safe_numeric_average\n\nlogger = logging.getLogger(__name__)\n\n\nclass PromptSampler:\n \"\"\"Generates prompts for code evolution\"\"\"\n\n def __init__(self, config: PromptConfig):\n self.config = config\n self.template_manager = TemplateManager(config.template_dir)\n\n # Initialize the random number generator\n random.seed()\n\n # Store custom template mappings\n self.system_template_override = None\n self.user_template_override = None\n\n logger.info(\"Initialized prompt sampler\")\n\n def set_templates(\n self, system_template: Optional[str] = None, user_template: Optional[str] = None\n ) -> None:\n \"\"\"\n Set custom templates to use for this sampler\n\n Args:\n system_template: Template name for system message\n user_template: Template name for user message\n \"\"\"\n self.system_template_override = system_template\n self.user_template_override = user_template\n logger.info(f\"Set custom templates: system={system_template}, user={user_template}\")\n\n def build_prompt(\n self,\n current_program: str = \"\",\n parent_program: str = \"\",\n program_metrics: Dict[str, float] = {},\n previous_programs: List[Dict[str, Any]] = [],\n top_programs: List[Dict[str, Any]] = [],\n language: str = \"python\",\n evolution_round: int = 0,\n allow_full_rewrite: bool = False,\n template_key: Optional[str] = \"diff_user\",\n program_artifacts: Optional[Dict[str, Union[str, bytes]]] = None,\n **kwargs: Any,\n ) -> Dict[str, str]:\n \"\"\"\n Build a prompt for the LLM\n\n Args:\n current_program: Current program code\n parent_program: Parent program from which current was derived\n program_metrics: Dictionary of metric names to values\n previous_programs: List of previous program attempts\n top_programs: List of top-performing programs\n language: Programming language\n evolution_round: Current evolution round\n allow_full_rewrite: Whether to allow a full rewrite\n template_key: Optional override for template key\n program_artifacts: Optional artifacts from program evaluation\n **kwargs: Additional keys to replace in the user prompt\n\n Returns:\n Dictionary with 'system' and 'user' keys\n \"\"\"\n # Select template based on whether we want a full rewrite (with overrides)\n if template_key:\n # Use explicitly provided template key\n user_template_key = template_key\n elif self.user_template_override:\n # Use the override set with set_templates\n user_template_key = self.user_template_override\n else:\n # Default behavior\n user_template_key = \"full_rewrite_user\" if allow_full_rewrite else \"diff_user\"\n\n # Get the template\n user_template = self.template_manager.get_template(user_template_key)\n\n # Use system template override if set\n if self.system_template_override:\n system_message = self.template_manager.get_template(self.system_template_override)\n else:\n system_message = self.config.system_message\n # If system_message is a template name rather than content, get the template\n if system_message in self.template_manager.templates:\n system_message = self.template_manager.get_template(system_message)\n\n # Format metrics\n metrics_str = self._format_metrics(program_metrics)\n\n # Identify areas for improvement\n improvement_areas = self._identify_improvement_areas(\n current_program, parent_program, program_metrics, previous_programs\n )\n\n # Format evolution history\n evolution_history = self._format_evolution_history(\n previous_programs, top_programs, language\n )\n\n # Format artifacts section if enabled and available\n artifacts_section = \"\"\n if self.config.include_artifacts and program_artifacts:\n artifacts_section = self._render_artifacts(program_artifacts)\n\n # Format proposal information\n current_proposal_str = self._format_proposal(kwargs.get('current_proposal', []))\n parent_proposal_str = self._format_proposal(kwargs.get('parent_proposal', []))\n proposal_score_str = f\"{kwargs.get('proposal_score', -1.0)}\"\n\n # Apply stochastic template variations if enabled\n if self.config.use_template_stochasticity:\n user_template = self._apply_template_variations(user_template)\n\n # Format the final user message\n user_message = user_template.format(\n parent_proposal_text=parent_proposal_str,\n language=language,\n parent_program=parent_program,\n metrics=metrics_str,\n improvement_areas=improvement_areas,\n artifacts=artifacts_section,\n evolution_history=evolution_history,\n current_proposal_text=current_proposal_str,\n **kwargs,\n )\n\n return {\n \"system\": system_message,\n \"user\": user_message,\n }\n\n def _format_metrics(self, metrics: Dict[str, float]) -> str:\n \"\"\"Format metrics for the prompt using safe formatting\"\"\"\n # Use safe formatting to handle mixed numeric and string values\n formatted_parts = []\n for name, value in metrics.items():\n if isinstance(value, (int, float)):\n try:\n formatted_parts.append(f\"- {name}: {value:.4f}\")\n except (ValueError, TypeError):\n formatted_parts.append(f\"- {name}: {value}\")\n else:\n formatted_parts.append(f\"- {name}: {value}\")\n return \"\\n\".join(formatted_parts)\n\n def _identify_improvement_areas(\n self,\n current_program: str,\n parent_program: str,\n metrics: Dict[str, float],\n previous_programs: List[Dict[str, Any]],\n ) -> str:\n \"\"\"Identify potential areas for improvement\"\"\"\n # This method could be expanded to include more sophisticated analysis\n # For now, we'll use a simple approach\n\n improvement_areas = []\n\n # Check program length\n if len(current_program) > 500:\n improvement_areas.append(\n \"Consider simplifying the code to improve readability and maintainability\"\n )\n\n # Check for performance patterns in previous attempts\n if len(previous_programs) >= 2:\n recent_attempts = previous_programs[-2:]\n metrics_improved = []\n metrics_regressed = []\n\n for metric, value in metrics.items():\n # Only compare numeric metrics\n if not isinstance(value, (int, float)) or isinstance(value, bool):\n continue\n\n improved = True\n regressed = True\n\n for attempt in recent_attempts:\n attempt_value = attempt[\"metrics\"].get(metric, 0)\n # Only compare if both values are numeric\n if isinstance(value, (int, float)) and isinstance(attempt_value, (int, float)):\n if attempt_value <= value:\n regressed = False\n if attempt_value >= value:\n improved = False\n else:\n # If either value is non-numeric, skip comparison\n improved = False\n regressed = False\n\n if improved and metric not in metrics_improved:\n metrics_improved.append(metric)\n if regressed and metric not in metrics_regressed:\n metrics_regressed.append(metric)\n\n if metrics_improved:\n improvement_areas.append(\n f\"Metrics showing improvement: {', '.join(metrics_improved)}. \"\n \"Consider continuing with similar changes.\"\n )\n\n if metrics_regressed:\n improvement_areas.append(\n f\"Metrics showing regression: {', '.join(metrics_regressed)}. \"\n \"Consider reverting or revising recent changes in these areas.\"\n )\n\n # If we don't have specific improvements to suggest\n if not improvement_areas:\n improvement_areas.append(\n \"Focus on optimizing the code for better performance on the target metrics\"\n )\n\n return \"\\n\".join([f\"- {area}\" for area in improvement_areas])\n\n def _format_evolution_history(\n self,\n previous_programs: List[Dict[str, Any]],\n top_programs: List[Dict[str, Any]],\n language: str,\n ) -> str:\n \"\"\"Format the evolution history for the prompt\"\"\"\n # Get templates\n history_template = self.template_manager.get_template(\"evolution_history\")\n previous_attempt_template = self.template_manager.get_template(\"previous_attempt\")\n top_program_template = self.template_manager.get_template(\"top_program\")\n\n # Format previous attempts (most recent first)\n previous_attempts_str = \"\"\n selected_previous = previous_programs[-min(3, len(previous_programs)) :]\n\n for i, program in enumerate(reversed(selected_previous)):\n attempt_number = len(previous_programs) - i\n changes = program.get(\"changes\", \"Unknown changes\")\n\n # Format performance metrics using safe formatting\n performance_parts = []\n for name, value in program.get(\"metrics\", {}).items():\n if isinstance(value, (int, float)):\n try:\n performance_parts.append(f\"{name}: {value:.4f}\")\n except (ValueError, TypeError):\n performance_parts.append(f\"{name}: {value}\")\n else:\n performance_parts.append(f\"{name}: {value}\")\n performance_str = \", \".join(performance_parts)\n\n # Determine outcome based on comparison with parent (only numeric metrics)\n parent_metrics = program.get(\"parent_metrics\", {})\n outcome = \"Mixed results\"\n\n # Safely compare only numeric metrics\n program_metrics = program.get(\"metrics\", {})\n\n # Check if all numeric metrics improved\n numeric_comparisons_improved = []\n numeric_comparisons_regressed = []\n\n for m in program_metrics:\n prog_value = program_metrics.get(m, 0)\n parent_value = parent_metrics.get(m, 0)\n\n # Only compare if both values are numeric\n if isinstance(prog_value, (int, float)) and isinstance(parent_value, (int, float)):\n if prog_value > parent_value:\n numeric_comparisons_improved.append(True)\n else:\n numeric_comparisons_improved.append(False)\n\n if prog_value < parent_value:\n numeric_comparisons_regressed.append(True)\n else:\n numeric_comparisons_regressed.append(False)\n\n # Determine outcome based on numeric comparisons\n if numeric_comparisons_improved and all(numeric_comparisons_improved):\n outcome = \"Improvement in all metrics\"\n elif numeric_comparisons_regressed and all(numeric_comparisons_regressed):\n outcome = \"Regression in all metrics\"\n\n previous_attempts_str += (\n previous_attempt_template.format(\n attempt_number=attempt_number,\n changes=changes,\n performance=performance_str,\n outcome=outcome,\n )\n + \"\\n\\n\"\n )\n\n # Format top programs\n top_programs_str = \"\"\n selected_top = top_programs[: min(self.config.num_top_programs, len(top_programs))]\n\n for i, program in enumerate(selected_top):\n # Extract a snippet (first 10 lines) for display\n program_code = program.get(\"code\", \"\")\n program_snippet = \"\\n\".join(program_code.split(\"\\n\")[:10])\n if len(program_code.split(\"\\n\")) > 10:\n program_snippet += \"\\n# ... (truncated for brevity)\"\n\n # Calculate a composite score using safe numeric average\n score = safe_numeric_average(program.get(\"metrics\", {}))\n\n # Extract key features (this could be more sophisticated)\n key_features = program.get(\"key_features\", [])\n if not key_features:\n key_features = []\n for name, value in program.get(\"metrics\", {}).items():\n if isinstance(value, (int, float)):\n try:\n key_features.append(f\"Performs well on {name} ({value:.4f})\")\n except (ValueError, TypeError):\n key_features.append(f\"Performs well on {name} ({value})\")\n else:\n key_features.append(f\"Performs well on {name} ({value})\")\n\n key_features_str = \", \".join(key_features)\n\n top_programs_str += (\n top_program_template.format(\n program_number=i + 1,\n score=f\"{score:.4f}\",\n language=language,\n program_snippet=program_snippet,\n key_features=key_features_str,\n )\n + \"\\n\\n\"\n )\n\n # Format diverse programs using num_diverse_programs config\n diverse_programs_str = \"\"\n if (\n self.config.num_diverse_programs > 0\n and len(top_programs) > self.config.num_top_programs\n ):\n # Skip the top programs we already included\n remaining_programs = top_programs[self.config.num_top_programs :]\n\n # Sample diverse programs from the remaining\n num_diverse = min(self.config.num_diverse_programs, len(remaining_programs))\n if num_diverse > 0:\n # Use random sampling to get diverse programs\n diverse_programs = random.sample(remaining_programs, num_diverse)\n\n diverse_programs_str += \"\\n\\n## Diverse Programs\\n\\n\"\n\n for i, program in enumerate(diverse_programs):\n # Extract a snippet (first 5 lines for diversity)\n program_code = program.get(\"code\", \"\")\n program_snippet = \"\\n\".join(program_code.split(\"\\n\")[:5])\n if len(program_code.split(\"\\n\")) > 5:\n program_snippet += \"\\n# ... (truncated)\"\n\n # Calculate a composite score using safe numeric average\n score = safe_numeric_average(program.get(\"metrics\", {}))\n\n # Extract key features\n key_features = program.get(\"key_features\", [])\n if not key_features:\n key_features = [\n f\"Alternative approach to {name}\"\n for name in list(program.get(\"metrics\", {}).keys())[\n :2\n ] # Just first 2 metrics\n ]\n\n key_features_str = \", \".join(key_features)\n\n diverse_programs_str += (\n top_program_template.format(\n program_number=f\"D{i + 1}\",\n score=f\"{score:.4f}\",\n language=language,\n program_snippet=program_snippet,\n key_features=key_features_str,\n )\n + \"\\n\\n\"\n )\n\n # Combine top and diverse programs\n combined_programs_str = top_programs_str + diverse_programs_str\n\n # Combine into full history\n return history_template.format(\n previous_attempts=previous_attempts_str.strip(),\n top_programs=combined_programs_str.strip(),\n )\n\n def _apply_template_variations(self, template: str) -> str:\n \"\"\"Apply stochastic variations to the template\"\"\"\n result = template\n\n # Apply variations defined in the config\n for key, variations in self.config.template_variations.items():\n if variations and f\"{{{key}}}\" in result:\n chosen_variation = random.choice(variations)\n result = result.replace(f\"{{{key}}}\", chosen_variation)\n\n return result\n\n def _render_artifacts(self, artifacts: Dict[str, Union[str, bytes]]) -> str:\n \"\"\"\n Render artifacts for prompt inclusion\n\n Args:\n artifacts: Dictionary of artifact name to content\n\n Returns:\n Formatted string for prompt inclusion (empty string if no artifacts)\n \"\"\"\n if not artifacts:\n return \"\"\n\n sections = []\n\n # Process all artifacts using .items()\n for key, value in artifacts.items():\n content = self._safe_decode_artifact(value)\n # Truncate if too long\n if len(content) > self.config.max_artifact_bytes:\n content = content[: self.config.max_artifact_bytes] + \"\\n... (truncated)\"\n\n sections.append(f\"### {key}\\n```\\n{content}\\n```\")\n\n if sections:\n return \"## Last Execution Output\\n\\n\" + \"\\n\\n\".join(sections)\n else:\n return \"\"\n\n def _safe_decode_artifact(self, value: Union[str, bytes]) -> str:\n \"\"\"\n Safely decode an artifact value to string\n\n Args:\n value: Artifact value (string or bytes)\n\n Returns:\n String representation of the value\n \"\"\"\n if isinstance(value, str):\n # Apply security filter if enabled\n if self.config.artifact_security_filter:\n return self._apply_security_filter(value)\n return value\n elif isinstance(value, bytes):\n try:\n decoded = value.decode(\"utf-8\", errors=\"replace\")\n if self.config.artifact_security_filter:\n return self._apply_security_filter(decoded)\n return decoded\n except Exception:\n return f\"<binary data: {len(value)} bytes>\"\n else:\n return str(value)\n\n def _apply_security_filter(self, text: str) -> str:\n \"\"\"\n Apply security filtering to artifact text\n\n Args:\n text: Input text\n\n Returns:\n Filtered text with potential secrets/sensitive info removed\n \"\"\"\n import re\n\n # Remove ANSI escape sequences\n ansi_escape = re.compile(r\"\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])\")\n filtered = ansi_escape.sub(\"\", text)\n\n # Basic patterns for common secrets (can be expanded)\n secret_patterns = [\n (r\"[A-Za-z0-9]{32,}\", \"<REDACTED_TOKEN>\"), # Long alphanumeric tokens\n (r\"sk-[A-Za-z0-9]{48}\", \"<REDACTED_API_KEY>\"), # OpenAI-style API keys\n (r\"password[=:]\\s*[^\\s]+\", \"password=<REDACTED>\"), # Password assignments\n (r\"token[=:]\\s*[^\\s]+\", \"token=<REDACTED>\"), # Token assignments\n ]\n\n for pattern, replacement in secret_patterns:\n filtered = re.sub(pattern, replacement, filtered, flags=re.IGNORECASE)\n\n return filtered\n\n def _format_proposal(self, proposal: List[str]) -> str:\n \"\"\"\n Format proposal for prompt inclusion\n \n Args:\n proposal: List of proposal strings\n \n Returns:\n Formatted string for prompt inclusion\n \"\"\"\n if not proposal:\n return \"\"\n \n # Join proposal parts with newlines\n proposal_text = \"\\n\".join(proposal)\n \n # Truncate if too long\n if len(proposal_text) > 2000:\n proposal_text = proposal_text[:2000] + \"...(truncated)\"\n \n return proposal_text\n\n def _format_proposal_score(self, proposal_scores: List[dict]) -> str:\n \"\"\"\n Format proposal scores for prompt inclusion\n \n Args:\n proposal_scores: List of dictionaries containing proposal scores\n \n Returns:\n Formatted string for prompt inclusion\n \"\"\"\n if not proposal_scores:\n return \"\"\n \n # Extract scores from the score dictionaries\n scores = []\n for score_dict in proposal_scores:\n if isinstance(score_dict, dict) and 'score' in score_dict:\n scores.append(str(score_dict['score']))\n elif isinstance(score_dict, (int, float)):\n scores.append(f\"{score_dict:.2f}\")\n else:\n scores.append(str(score_dict))\n \n return \", \".join(scores)\n"}}, "reward_model/train/train.py": {"75cecaafc6e85ca65cf545714b6ef7779537c78e": {"t": 1785599215.9572818, "n": 5860, "h": "1a461916d45f2ee0", "c": "# This code is based on tatsu-lab/stanford_alpaca (https://github.com/tatsu-lab/stanford_alpaca).\r\n\r\nfrom dataclasses import dataclass, field\r\nimport math\r\nimport pathlib\r\nfrom typing import Dict, Optional\r\n\r\nimport transformers\r\nfrom transformers import Trainer\r\nfrom transformers.trainer_pt_utils import LabelSmoother\r\nfrom transformers import set_seed\r\n\r\nfrom preprocess import load_dataset, make_supervised_data_module, DataCollatorForSupervisedDataset\r\n\r\nIGNORE_TOKEN_ID = LabelSmoother.ignore_index\r\n\r\nIGNORE_INDEX = -100\r\nDEFAULT_PAD_TOKEN = \"[PAD]\"\r\nDEFAULT_EOS_TOKEN = \"</s>\"\r\nDEFAULT_BOS_TOKEN = \"<s>\"\r\nDEFAULT_UNK_TOKEN = \"<unk>\"\r\n\r\nset_seed(42)\r\n\r\n@dataclass\r\nclass ModelArguments:\r\n model_name_or_path: Optional[str] = field(default=\"facebook/opt-125m\")\r\n \r\n\r\n@dataclass\r\nclass DataArguments:\r\n data_path: str = field(\r\n default=None, metadata={\"help\": \"Path to the training data.\"}\r\n )\r\n eval_data_path: str = field(\r\n default=None, metadata={\"help\": \"Path to the evaluation data.\"}\r\n )\r\n lazy_preprocess: bool = True\r\n\r\n\r\n@dataclass\r\nclass TrainingArguments(transformers.TrainingArguments):\r\n cache_dir: Optional[str] = field(default=None)\r\n optim: str = field(default=\"adamw_torch\")\r\n model_max_length: int = field(\r\n default=512,\r\n metadata={\r\n \"help\": \"Maximum sequence length. Sequences will be right padded (and possibly truncated).\"\r\n },\r\n )\r\n\r\n\r\ndef trainer_save_model_safe(trainer: transformers.Trainer):\r\n from torch.distributed.fsdp import FullyShardedDataParallel as FSDP\r\n from torch.distributed.fsdp import StateDictType, FullStateDictConfig\r\n\r\n save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True)\r\n with FSDP.state_dict_type(\r\n trainer.model, StateDictType.FULL_STATE_DICT, save_policy\r\n ):\r\n trainer.save_model()\r\n\r\n\r\ndef smart_tokenizer_and_embedding_resize(\r\n special_tokens_dict: Dict,\r\n tokenizer: transformers.PreTrainedTokenizer,\r\n model: transformers.PreTrainedModel,\r\n):\r\n \"\"\"Resize tokenizer and embedding.\r\n\r\n Note: This is the unoptimized version that may make your embedding size not be divisible by 64.\r\n \"\"\"\r\n num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)\r\n model.resize_token_embeddings(len(tokenizer))\r\n\r\n if num_new_tokens > 0:\r\n input_embeddings = model.get_input_embeddings().weight.data\r\n output_embeddings = model.get_output_embeddings().weight.data\r\n\r\n input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(\r\n dim=0, keepdim=True\r\n )\r\n output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(\r\n dim=0, keepdim=True\r\n )\r\n\r\n input_embeddings[-num_new_tokens:] = input_embeddings_avg\r\n output_embeddings[-num_new_tokens:] = output_embeddings_avg\r\n\r\n\r\ndef train():\r\n global local_rank\r\n\r\n parser = transformers.HfArgumentParser(\r\n (ModelArguments, DataArguments, TrainingArguments)\r\n )\r\n model_args, data_args, training_args = parser.parse_args_into_dataclasses()\r\n local_rank = training_args.local_rank\r\n\r\n # Set RoPE scaling factor\r\n config = transformers.AutoConfig.from_pretrained(\r\n model_args.model_name_or_path,\r\n cache_dir=training_args.cache_dir,\r\n )\r\n orig_ctx_len = getattr(config, \"max_position_embeddings\", None)\r\n if orig_ctx_len and training_args.model_max_length > orig_ctx_len:\r\n scaling_factor = float(math.ceil(training_args.model_max_length / orig_ctx_len))\r\n config.rope_scaling = {\"type\": \"linear\", \"factor\": scaling_factor}\r\n config.use_cache = False\r\n\r\n # Load model and tokenizer\r\n tokenizer = transformers.AutoTokenizer.from_pretrained(\r\n model_args.model_name_or_path,\r\n cache_dir=training_args.cache_dir,\r\n model_max_length = training_args.model_max_length,\r\n truncation = True,\r\n padding_side = \"right\",\r\n trust_remote_code = True,\r\n use_fast=True,\r\n )\r\n\r\n # Load data\r\n if '.json' in data_args.data_path:\r\n data_module = make_supervised_data_module(tokenizer=tokenizer, data_path=data_args.data_path)\r\n else:\r\n train_dataset = load_dataset(data_args.data_path)\r\n data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)\r\n data_module = dict(\r\n train_dataset=train_dataset, eval_dataset=None, data_collator=data_collator\r\n )\r\n \r\n model = transformers.AutoModelForCausalLM.from_pretrained(\r\n model_args.model_name_or_path,\r\n config=config,\r\n cache_dir=training_args.cache_dir,\r\n use_flash_attention_2=True\r\n )\r\n\r\n if local_rank == 0:\r\n print(config)\r\n print(model)\r\n \r\n # tokenizer.pad_token = tokenizer.unk_token\r\n special_tokens_dict = dict()\r\n if tokenizer.pad_token is None:\r\n special_tokens_dict[\"pad_token\"] = DEFAULT_PAD_TOKEN\r\n if tokenizer.eos_token is None:\r\n special_tokens_dict[\"eos_token\"] = DEFAULT_EOS_TOKEN\r\n if tokenizer.bos_token is None:\r\n special_tokens_dict[\"bos_token\"] = DEFAULT_BOS_TOKEN\r\n if tokenizer.unk_token is None:\r\n special_tokens_dict[\"unk_token\"] = DEFAULT_UNK_TOKEN\r\n\r\n smart_tokenizer_and_embedding_resize(\r\n special_tokens_dict=special_tokens_dict,\r\n tokenizer=tokenizer,\r\n model=model,\r\n )\r\n\r\n # Start trainner\r\n trainer = Trainer(\r\n model=model, tokenizer=tokenizer, args=training_args, **data_module\r\n )\r\n if list(pathlib.Path(training_args.output_dir).glob(\"checkpoint-*\")):\r\n trainer.train(resume_from_checkpoint=True)\r\n else:\r\n trainer.train()\r\n\r\n # Save model\r\n model.config.use_cache = True\r\n trainer.save_state()\r\n trainer_save_model_safe(trainer)\r\n\r\n\r\nif __name__ == \"__main__\":\r\n train()\r\n"}}, "run.py": {"ee66703373f39f04eb85b765e17a12bb1dca961a": {"t": 1785599215.9861646, "n": 655, "h": "c4bf64d678293189", "c": "from evolve_agent import EvolveAgent \nimport asyncio\nimport logging\nimport os\n\nos.environ['CUDA_VISIBLE_DEVICES'] = '6'\n# logging.basicConfig(\n# level=logging.DEBUG, \n# # format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', \n# # filename='app.log', \n# )\n\nevolve_agent = EvolveAgent(\n initial_program_path=\"results/initial_program.py\",\n evaluation_file=\"results/evaluator.py\",\n initial_proposal_path=\"results/initial_proposal.txt\",\n config_path=\"configs/oai_config.yaml\"\n)\n\nasync def main():\n best_program = await evolve_agent.run(iterations=50) \n print(best_program)\n\nasyncio.run(main())\n# print(evolve_agent)\n\n"}}, "reward_model/grade.py": {"ee696247c739f7087e2c5e1c051ef30170bf87c8": {"t": 1785599216.0729833, "n": 4154, "h": "aa772cd14760d62d", "c": "import json\nimport numpy as np\nimport pandas as pd\nfrom typing import List, Dict\nimport re\nfrom vllm import LLM, SamplingParams\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\nfrom llm.backend import score_abstracts_with_vllm, score_abstracts_with_api\n\n\ndef evaluate_and_compare(data: List[Dict], model_results: List[Dict]) -> Dict:\n results = {\n \"model_scores\": [res[\"score\"] for res in model_results],\n \"avg_ratings\": [res[\"avg_rating\"] for res in model_results],\n \"evaluations\": [res[\"evaluation\"] for res in model_results],\n \"abstracts\": [res[\"abstract\"] for res in model_results],\n \"differences\": [],\n \"mae\": 0.0,\n \"mse\": 0.0,\n \"accuracy\": 0.0 # New metric for accuracy\n }\n\n # Calculate differences and labels\n valid_differences = []\n true_labels = []\n pred_labels = []\n for ms, ar in zip(results[\"model_scores\"], results[\"avg_ratings\"]):\n # Calculate difference\n diff = abs(ms - ar) if ms >= 0 else -1\n valid_differences.append(diff)\n\n # Assign labels: positive (1) if score > 5.5, negative (0) otherwise\n # Only include valid scores for accuracy calculation\n if ms >= 0:\n true_label = 1 if ar > 5.5 else 0\n pred_label = 1 if ms > 5.5 else 0\n true_labels.append(true_label)\n pred_labels.append(pred_label)\n\n # Calculate metrics\n results[\"differences\"] = valid_differences\n valid_diffs = [d for d in valid_differences if d >= 0]\n results[\"mae\"] = np.mean(valid_diffs) if valid_diffs else 0.0\n results[\"mse\"] = np.mean([d ** 2 for d in valid_diffs]) if valid_diffs else 0.0\n \n # Calculate accuracy: proportion of matching labels\n correct_predictions = sum(1 for t, p in zip(true_labels, pred_labels) if t == p)\n results[\"accuracy\"] = correct_predictions / len(true_labels) if true_labels else 0.0\n\n return results\n\ndef print_results(results: Dict):\n print(f\"{'Index':<6} {'Model Score':<12} {'Avg Rating':<12} {'Difference':<12} {'Evaluation':<50}\")\n print(\"-\" * 100)\n for i in range(len(results[\"model_scores\"])):\n eval_snippet = results[\"evaluations\"][i][:47] + \"...\" if len(results[\"evaluations\"][i]) > 47 else results[\"evaluations\"][i]\n diff = results[\"differences\"][i] if results[\"differences\"][i] >= 0 else \"N/A\"\n print(f\"{i+1:<6} {results['model_scores'][i]:<12.2f} {results['avg_ratings'][i]:<12.2f} {diff:<12} {eval_snippet:<50}\")\n print(\"\\nSummary Statistics (excluding invalid scores):\")\n print(f\"Mean Absolute Error (MAE): {results['mae']:.2f}\")\n print(f\"Mean Squared Error (MSE): {results['mse']:.2f}\")\n print(f\"Prediction Accuracy: {results['accuracy']:.2%}\") # Display accuracy as percentage\n\ndef main():\n\n try:\n data = load_dataset('json', data_files='/data/zhuotaodeng/yzj/alpha-research/data/iclr2025_eval_100.json', split='train')\n data = [dict(item) for item in data] # Convert to List[Dict]\n except Exception as e:\n print(f\"Error loading dataset: {e}\")\n return\n\n model_results = score_abstracts_with_vllm(data, model_name=\"/data/zhuotaodeng/yzj/download_from_modelscope/Qwen/Qwen3-8B\")\n # model_results = score_abstracts_with_vllm(data, '/data/zhuotaodeng/yzj/alpha-research/model/qwen25_grm_iclr_boxed/checkpoint-120')\n # model_results = score_abstracts_with_api(data, '/data/zhuotaodeng/yzj/alpha-research/idea-eval/results.jsonl')\n\n results = evaluate_and_compare(data, model_results)\n\n print_results(results)\n\n with open(\"vllm_evaluation_results.json\", \"w\") as f:\n json.dump(results, f, indent=4)\n print(\"\\nResults saved to 'vllm_evaluation_results.json'\")\n\n df = pd.DataFrame({\n \"Abstract\": results[\"abstracts\"],\n \"Model_Score\": results[\"model_scores\"],\n \"Avg_Rating\": results[\"avg_ratings\"],\n \"Difference\": results[\"differences\"],\n \"Evaluation\": results[\"evaluations\"]\n })\n df.to_csv(\"vllm_evaluation_results.csv\", index=False)\n print(\"Results also saved to 'vllm_evaluation_results.csv'\")\n\n\nif __name__ == \"__main__\":\n main()"}}, "reward_model/train/script/train_qwen.sh": {"18c2c59c9301e0ddba3bd364a3bb1ed344339d43": {"t": 1785599216.1861415, "n": 1116, "h": "fadaa3d4b0f0457f", "c": "export CUDA_VISIBLE_DEVICES=3,2\nexport WANDB_DISABLED=true\nset -ex\nLOG_PATH=./log.txt\nSAVE_PATH=/data/zhuotaodeng/yzj/alpha-research/model/qwen25_grm_iclr_boxed\nmkdir -p $SAVE_PATH\n\ntorchrun --nproc_per_node=2 \\\n --master_port=20011 \\\n train.py \\\n --model_name_or_path /data/zhuotaodeng/yzj/download_from_modelscope/Qwen/Qwen2___5-7B-Instruct \\\n --data_path /data/zhuotaodeng/yzj/alpha-research/data/iclr_train_all_boxed.json \\\n --bf16 True \\\n --tf32 True \\\n --output_dir $SAVE_PATH \\\n --num_train_epochs 2 \\\n --per_device_train_batch_size 2 \\\n --per_device_eval_batch_size 2 \\\n --gradient_accumulation_steps 32 \\\n --evaluation_strategy \"no\" \\\n --save_strategy \"steps\" \\\n --save_steps 60 \\\n --save_total_limit 17 \\\n --learning_rate 1e-5 \\\n --weight_decay 0. \\\n --warmup_steps 100 \\\n --lr_scheduler_type \"cosine\" \\\n --logging_steps 1 \\\n --fsdp \"full_shard auto_wrap\" \\\n --fsdp_transformer_layer_cls_to_wrap 'Qwen2DecoderLayer' \\\n --model_max_length 16384 \\\n --gradient_checkpointing True \\\n --lazy_preprocess False\n \n > $LOG_PATH 2>&1\n"}}, "reward_model/train/utils.py": {"29d249930c0b6f6d1f34f77c84c300497f044345": {"t": 1785599216.25518, "n": 6654, "h": "cea5289c8dc3acf5", "c": "import dataclasses\r\nimport logging\r\nimport math\r\nimport os\r\nimport io\r\nimport sys\r\nimport time\r\nimport json\r\nfrom typing import Optional, Sequence, Union\r\n\r\nimport openai\r\nimport tqdm\r\nfrom openai import openai_object\r\nimport copy\r\n\r\nStrOrOpenAIObject = Union[str, openai_object.OpenAIObject]\r\n\r\nopenai_org = os.getenv(\"OPENAI_ORG\")\r\nif openai_org is not None:\r\n openai.organization = openai_org\r\n logging.warning(f\"Switching to organization: {openai_org} for OAI API key.\")\r\n\r\n\r\n@dataclasses.dataclass\r\nclass OpenAIDecodingArguments(object):\r\n max_tokens: int = 1800\r\n temperature: float = 0.2\r\n top_p: float = 1.0\r\n n: int = 1\r\n stream: bool = False\r\n stop: Optional[Sequence[str]] = None\r\n presence_penalty: float = 0.0\r\n frequency_penalty: float = 0.0\r\n suffix: Optional[str] = None\r\n logprobs: Optional[int] = None\r\n echo: bool = False\r\n\r\n\r\ndef openai_completion(\r\n prompts: Union[str, Sequence[str], Sequence[dict[str, str]], dict[str, str]],\r\n decoding_args: OpenAIDecodingArguments,\r\n model_name=\"text-davinci-003\",\r\n sleep_time=2,\r\n batch_size=1,\r\n max_instances=sys.maxsize,\r\n max_batches=sys.maxsize,\r\n return_text=False,\r\n **decoding_kwargs,\r\n) -> Union[Union[StrOrOpenAIObject], Sequence[StrOrOpenAIObject], Sequence[Sequence[StrOrOpenAIObject]],]:\r\n \"\"\"Decode with OpenAI API.\r\n\r\n Args:\r\n prompts: A string or a list of strings to complete. If it is a chat model the strings should be formatted\r\n as explained here: https://github.com/openai/openai-python/blob/main/chatml.md. If it is a chat model\r\n it can also be a dictionary (or list thereof) as explained here:\r\n https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb\r\n decoding_args: Decoding arguments.\r\n model_name: Model name. Can be either in the format of \"org/model\" or just \"model\".\r\n sleep_time: Time to sleep once the rate-limit is hit.\r\n batch_size: Number of prompts to send in a single request. Only for non chat model.\r\n max_instances: Maximum number of prompts to decode.\r\n max_batches: Maximum number of batches to decode. This argument will be deprecated in the future.\r\n return_text: If True, return text instead of full completion object (which contains things like logprob).\r\n decoding_kwargs: Additional decoding arguments. Pass in `best_of` and `logit_bias` if you need them.\r\n\r\n Returns:\r\n A completion or a list of completions.\r\n Depending on return_text, return_openai_object, and decoding_args.n, the completion type can be one of\r\n - a string (if return_text is True)\r\n - an openai_object.OpenAIObject object (if return_text is False)\r\n - a list of objects of the above types (if decoding_args.n > 1)\r\n \"\"\"\r\n is_single_prompt = isinstance(prompts, (str, dict))\r\n if is_single_prompt:\r\n prompts = [prompts]\r\n\r\n if max_batches < sys.maxsize:\r\n logging.warning(\r\n \"`max_batches` will be deprecated in the future, please use `max_instances` instead.\"\r\n \"Setting `max_instances` to `max_batches * batch_size` for now.\"\r\n )\r\n max_instances = max_batches * batch_size\r\n\r\n prompts = prompts[:max_instances]\r\n num_prompts = len(prompts)\r\n prompt_batches = [\r\n prompts[batch_id * batch_size : (batch_id + 1) * batch_size]\r\n for batch_id in range(int(math.ceil(num_prompts / batch_size)))\r\n ]\r\n\r\n completions = []\r\n for batch_id, prompt_batch in tqdm.tqdm(\r\n enumerate(prompt_batches),\r\n desc=\"prompt_batches\",\r\n total=len(prompt_batches),\r\n ):\r\n batch_decoding_args = copy.deepcopy(decoding_args) # cloning the decoding_args\r\n\r\n while True:\r\n try:\r\n shared_kwargs = dict(\r\n model=model_name,\r\n **batch_decoding_args.__dict__,\r\n **decoding_kwargs,\r\n )\r\n completion_batch = openai.Completion.create(prompt=prompt_batch, **shared_kwargs)\r\n choices = completion_batch.choices\r\n\r\n for choice in choices:\r\n choice[\"total_tokens\"] = completion_batch.usage.total_tokens\r\n completions.extend(choices)\r\n break\r\n except openai.error.OpenAIError as e:\r\n logging.warning(f\"OpenAIError: {e}.\")\r\n if \"Please reduce your prompt\" in str(e):\r\n batch_decoding_args.max_tokens = int(batch_decoding_args.max_tokens * 0.8)\r\n logging.warning(f\"Reducing target length to {batch_decoding_args.max_tokens}, Retrying...\")\r\n else:\r\n logging.warning(\"Hit request rate limit; retrying...\")\r\n time.sleep(sleep_time) # Annoying rate limit on requests.\r\n\r\n if return_text:\r\n completions = [completion.text for completion in completions]\r\n if decoding_args.n > 1:\r\n # make completions a nested list, where each entry is a consecutive decoding_args.n of original entries.\r\n completions = [completions[i : i + decoding_args.n] for i in range(0, len(completions), decoding_args.n)]\r\n if is_single_prompt:\r\n # Return non-tuple if only 1 input and 1 generation.\r\n (completions,) = completions\r\n return completions\r\n\r\n\r\ndef _make_w_io_base(f, mode: str):\r\n if not isinstance(f, io.IOBase):\r\n f_dirname = os.path.dirname(f)\r\n if f_dirname != \"\":\r\n os.makedirs(f_dirname, exist_ok=True)\r\n f = open(f, mode=mode)\r\n return f\r\n\r\n\r\ndef _make_r_io_base(f, mode: str):\r\n if not isinstance(f, io.IOBase):\r\n f = open(f, mode=mode)\r\n return f\r\n\r\n\r\ndef jdump(obj, f, mode=\"w\", indent=4, default=str):\r\n \"\"\"Dump a str or dictionary to a file in json format.\r\n\r\n Args:\r\n obj: An object to be written.\r\n f: A string path to the location on disk.\r\n mode: Mode for opening the file.\r\n indent: Indent for storing json dictionaries.\r\n default: A function to handle non-serializable entries; defaults to `str`.\r\n \"\"\"\r\n f = _make_w_io_base(f, mode)\r\n if isinstance(obj, (dict, list)):\r\n json.dump(obj, f, indent=indent, default=default)\r\n elif isinstance(obj, str):\r\n f.write(obj)\r\n else:\r\n raise ValueError(f\"Unexpected type: {type(obj)}\")\r\n f.close()\r\n\r\n\r\ndef jload(f, mode=\"r\"):\r\n \"\"\"Load a .json file into a dictionary.\"\"\"\r\n f = _make_r_io_base(f, mode)\r\n jdict = json.load(f)\r\n f.close()\r\n return jdict\r\n"}}, "reward_model/llm/backend.py": {"4a0852ed1327000625314607ac3f0a0144ddaf8f": {"t": 1785599216.4905083, "n": 9665, "h": "7e58183b90fdafa0", "c": "import re\nimport os\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nfrom typing import List, Dict\n\nfrom vllm import LLM, SamplingParams\nfrom openai import OpenAI\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\n\n# SCORING_PROMPT = f\"\"\"\n# You are an expert reviewer tasked with evaluating the quality of a research abstract. \n# Your goal is to assign a score between 1 and 10 based on the abstract's clarity, novelty, technical rigor, and potential impact. Here are the criteria:\n# 1. Read the following abstract carefully and provide a score from 1 to 10. \n# 2. Score 6 means slightly higher than the boardline, 5 is slightly lower than the boardline.\n# Write the score in the {BOX}.\n# **idea**:\n# \"\"\"\n\n\nBOX=r\"\\boxed{}\"\nSYSTEM_PROMPT = \"You are an expert reviewer tasked with evaluating the quality of a research proposal. \"\nSCORING_PROMPT = f\"\"\"\nYour goal is to assign a score between 1 and 10 based on the proposal's clarity, novelty, technical rigor, and potential impact. Here are the criteria:\n1. Read the following proposal carefully and provide a score from 1 to 10. \n2. Score 6 means slightly higher than the boardline, 5 is slightly lower than the boardline.\nWrite the score in the {BOX}.\n**idea**:\n\n\"\"\"\n\ndef parse_score_from_text(text: str) -> float:\n match = re.search(r'\\\\boxed\\{(\\d*\\.?\\d*)\\}', text)\n if match:\n try:\n score = float(match.group(1))\n if 0 <= score <= 10:\n return score\n except ValueError:\n pass\n return -1.0 \n\n\ndef score_abstracts_with_vllm(data: List[Dict], model_name: str) -> List[Dict]:\n\n llm = LLM(model=model_name, gpu_memory_utilization=0.95)\n tokenizer = AutoTokenizer.from_pretrained(model_name)\n\n prompts = [\n tokenizer.apply_chat_template(\n [\n {\n \"role\": \"system\",\n \"content\": SYSTEM_PROMPT\n },\n {\n \"role\": \"user\",\n \"content\": SCORING_PROMPT + item[\"title\"] + \"\\n\" + item[\"abstract\"]\n }\n ],\n tokenize=False,\n add_generation_prompt=True,\n enable_thinking=False \n )\n for item in data\n ]\n \n sampling_params = SamplingParams(\n temperature=0,\n top_p=1.0,\n max_tokens=1000, \n )\n\n outputs = llm.generate(prompts, sampling_params)\n\n print(prompts[0])\n print(outputs[0].outputs[0].text)\n\n results = []\n for output, item in zip(outputs, data):\n output_text = output.outputs[0].text.strip()\n score = parse_score_from_text(output_text)\n results.append({\n \"score\": score,\n \"evaluation\": output_text,\n \"abstract\": item[\"abstract\"],\n \"avg_rating\": item[\"avg_rating\"]\n })\n\n return results\n\ndef load_processed_ids(jsonl_file: str) -> set:\n \"\"\"\n Loads the IDs of already processed abstracts from the JSONL file.\n\n Args:\n jsonl_file: Path to the JSONL file.\n\n Returns:\n Set of processed abstract titles.\n \"\"\"\n processed_ids = set()\n if os.path.exists(jsonl_file):\n with open(jsonl_file, 'r', encoding='utf-8') as f:\n for line in f:\n try:\n data = json.loads(line.strip())\n if data.get('score', -1.0) != -1.0: # Only include valid scores\n processed_ids.add(data['title'])\n except json.JSONDecodeError:\n print(f\"Warning: Skipping invalid JSON line in {jsonl_file}\")\n return processed_ids\n\ndef write_result_to_jsonl(result: Dict, jsonl_file: str):\n \"\"\"\n Writes a single result to the JSONL file if the score is valid.\n\n Args:\n result: Dictionary containing the result to write.\n jsonl_file: Path to the JSONL file.\n \"\"\"\n if result['score'] != -1.0: # Only write valid scores\n with open(jsonl_file, 'a', encoding='utf-8') as f:\n f.write(json.dumps(result, ensure_ascii=False) + '\\n')\n\ndef score_abstracts_with_api(data: List[Dict], \n jsonl_file: str, \n model_name: str = \"deepseek-chat\", \n api_key: str = \"sk-2c3f1f58031b4b86afdb6a8192ea02e2\", \n base_url: str = \"https://api.deepseek.com\",\n max_retries: int = 50, \n retry_delay: int = 5\n ) -> List[Dict]:\n \"\"\"\n Scores research proposals using OpenAI's API, writing valid results to a JSONL file incrementally.\n Resumes from the last successfully processed abstract (with valid score) in the JSONL file.\n\n Args:\n data: List of dictionaries containing 'title', 'abstract', and 'avg_rating'.\n jsonl_file: Path to the JSONL file for storing results.\n model_name: OpenAI model to use (default: 'gpt-4o').\n api_key: OpenAI API key (if not set, assumes it's configured in environment).\n max_retries: Maximum number of retries per abstract (default: 5).\n retry_delay: Seconds to wait between retries (default: 5).\n\n Returns:\n List of dictionaries with 'title', 'score', 'evaluation', 'abstract', and 'avg_rating'.\n \"\"\"\n # Initialize OpenAI client\n client = OpenAI(api_key=api_key, base_url=base_url)\n\n # Load already processed abstracts (with valid scores) to skip them\n processed_ids = load_processed_ids(jsonl_file)\n results = []\n\n # Filter out already processed abstracts\n data_to_process = [item for item in data if item['title'] not in processed_ids]\n print(f\"Total abstracts: {len(data)}, To process: {len(data_to_process)}, Already processed: {len(processed_ids)}\")\n\n # Prepare prompts for remaining abstracts\n prompts = [\n [\n {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": SCORING_PROMPT + item[\"title\"] + \"\\n\" + item[\"abstract\"]}\n ]\n for item in data_to_process\n ]\n\n for prompt, item in zip(prompts, data_to_process):\n retries = 0\n score = -1.0\n output_text = \"\"\n\n # Keep retrying until a valid score is obtained or max_retries is reached\n while score == -1.0 and retries < max_retries:\n response = client.chat.completions.create(\n model=model_name,\n messages=prompt,\n temperature=0,\n max_tokens=1000,\n top_p=1.0\n )\n output_text = response.choices[0].message.content.strip()\n score = parse_score_from_text(output_text)\n \n if score == -1.0:\n retries += 1\n print(f\"Invalid score for abstract: {item['title']}, Retry {retries}/{max_retries}\")\n time.sleep(retry_delay) # Wait before retrying\n else:\n print(f\"Prompt: {prompt}\")\n print(f\"Output: {output_text}\")\n\n # Create result dictionary\n result = {\n \"title\": item[\"title\"],\n \"score\": score,\n \"evaluation\": output_text,\n \"abstract\": item[\"abstract\"],\n \"avg_rating\": item[\"avg_rating\"]\n }\n\n # Write result to JSONL file only if score is valid\n write_result_to_jsonl(result, jsonl_file)\n results.append(result)\n\n if score == -1.0:\n print(f\"Failed to get valid score for abstract: {item['title']} after {max_retries} retries\")\n\n # Load previously processed results from JSONL to include in return\n if processed_ids:\n with open(jsonl_file, 'r', encoding='utf-8') as f:\n for line in f:\n try:\n result = json.loads(line.strip())\n if result['title'] in processed_ids:\n results.append(result)\n except json.JSONDecodeError:\n print(f\"Warning: Skipping invalid JSON line in {jsonl_file}\")\n\n return results\n\nif __name__ == '__main__':\n abst = \"\"\"\nTest-time scaling is a promising new approach to language modeling that uses extra test-time compute to improve performance. Recently, OpenAI's o1 model showed this capability but did not publicly share its methodology, leading to many replication efforts. We seek the simplest approach to achieve test-time scaling and strong reasoning performance. First, we curate a small dataset s1K of 1,000 questions paired with reasoning traces relying on three criteria we validate through ablations: difficulty, diversity, and quality. Second, we develop budget forcing to control test-time compute by forcefully terminating the model's thinking process or lengthening it by appending \"Wait\" multiple times to the model's generation when it tries to end. This can lead the model to double-check its answer, often fixing incorrect reasoning steps. After supervised finetuning the Qwen2.5-32B-Instruct language model on s1K and equipping it with budget forcing, our model s1-32B exceeds o1-preview on competition math questions by up to 27% (MATH and AIME24). Further, scaling s1-32B with budget forcing allows extrapolating beyond its performance without test-time intervention: from 50% to 57% on AIME24. \"\"\"\n title = \"s1: Simple test-time scaling\"\n data = [{\"title\": title, \"abstract\": abst, \"avg_rating\": 0}]\n print(score_abstracts_with_vllm(data, '/data/zhuotaodeng/yzj/alpha-research/model/qwen25_grm_iclr/checkpoint-240'))\n # print(score_abstracts_with_vllm(data, '/data/zhuotaodeng/yzj/download_from_modelscope/Qwen/Qwen3-8B'))"}}, "evolve_agent/controller.py": {"693e15675e37c1a92cfbd647b7280513ca84874e": {"t": 1785599216.518801, "n": 31497, "h": "c54efd6c486f5118", "c": "\"\"\"\nMain controller for EvolveAgent\n\"\"\"\n\nimport asyncio\nimport logging\nimport os\nimport re\nimport time\nimport uuid\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Tuple, Union\nimport traceback\n\nfrom evolve_agent.config import Config, load_config\nfrom evolve_agent.database import Program, ProgramDatabase\nfrom evolve_agent.evaluator import Evaluator\nfrom evolve_agent.llm.ensemble import LLMEnsemble\nfrom evolve_agent.prompt.sampler import PromptSampler\nfrom evolve_agent.reward_model import RewardModel\nfrom evolve_agent.utils.code_utils import (\n apply_diff,\n extract_code_language,\n extract_diffs,\n format_diff_summary,\n parse_evolve_blocks,\n parse_full_rewrite,\n)\nfrom evolve_agent.utils.format_utils import (\n format_metrics_safe,\n format_improvement_safe,\n)\n\nlogger = logging.getLogger(__name__)\n\ndef _format_metrics(metrics: Dict[str, Any]) -> str:\n \"\"\"Safely format metrics, handling both numeric and string values\"\"\"\n formatted_parts = []\n for name, value in metrics.items():\n if isinstance(value, (int, float)) and not isinstance(value, bool):\n try:\n formatted_parts.append(f\"{name}={value:.4f}\")\n except (ValueError, TypeError):\n formatted_parts.append(f\"{name}={value}\")\n else:\n formatted_parts.append(f\"{name}={value}\")\n return \", \".join(formatted_parts)\n\n\ndef _format_improvement(improvement: Dict[str, Any]) -> str:\n \"\"\"Safely format improvement metrics\"\"\"\n formatted_parts = []\n for name, diff in improvement.items():\n if isinstance(diff, (int, float)) and not isinstance(diff, bool):\n try:\n formatted_parts.append(f\"{name}={diff:+.4f}\")\n except (ValueError, TypeError):\n formatted_parts.append(f\"{name}={diff}\")\n else:\n formatted_parts.append(f\"{name}={diff}\")\n return \", \".join(formatted_parts)\n\n\nclass EvolveAgent:\n \"\"\"\n Main controller for EvolveAgent\n\n Orchestrates the evolution process, coordinating between the prompt sampler,\n LLM ensemble, evaluator, and program database.\n\n Features:\n - Tracks the absolute best program across evolution steps\n - Ensures the best solution is not lost during the MAP-Elites process\n - Always includes the best program in the selection process for inspiration\n - Maintains detailed logs and metadata about improvements\n \"\"\"\n\n def __init__(\n self,\n initial_program_path: str,\n initial_proposal_path: str,\n evaluation_file: str,\n config_path: Optional[str] = None,\n config: Optional[Config] = None,\n output_dir: Optional[str] = None,\n ):\n # Load configuration\n if config is not None:\n # Use provided Config object directly\n self.config = config\n else:\n # Load from file or use defaults\n self.config = load_config(config_path)\n\n # Set up output directory\n self.output_dir = output_dir or os.path.join(\n os.path.dirname(initial_program_path), \"evolve_agent_output\"\n )\n os.makedirs(self.output_dir, exist_ok=True)\n\n # Set up logging\n self._setup_logging()\n\n # Set random seed for reproducibility if specified\n if self.config.random_seed is not None:\n import random\n import numpy as np\n\n random.seed(self.config.random_seed)\n np.random.seed(self.config.random_seed)\n logger.info(f\"Set random seed to {self.config.random_seed} for reproducibility\")\n\n # Load initial program\n self.initial_program_path = initial_program_path\n self.initial_program_code = self._load_initial_program()\n self.language = extract_code_language(self.initial_program_code)\n\n self.initial_proposal_path = initial_proposal_path\n self.initial_proposal = self._load_initial_proposal()\n\n # Extract file extension from initial program\n self.file_extension = os.path.splitext(initial_program_path)[1]\n if not self.file_extension:\n # Default to .py if no extension found\n self.file_extension = \".py\"\n else:\n # Make sure it starts with a dot\n if not self.file_extension.startswith(\".\"):\n self.file_extension = f\".{self.file_extension}\"\n\n # Initialize components\n self.llm_ensemble = LLMEnsemble(self.config.llm.models)\n self.llm_evaluator_ensemble = LLMEnsemble(self.config.llm.evaluator_models)\n self.reward_model = RewardModel(self.config.rewardmodel)\n\n self.prompt_sampler = PromptSampler(self.config.prompt)\n self.evaluator_prompt_sampler = PromptSampler(self.config.prompt)\n self.evaluator_prompt_sampler.set_templates(\"evaluator_system_message\")\n\n # Pass random seed to database if specified\n if self.config.random_seed is not None:\n self.config.database.random_seed = self.config.random_seed\n\n self.database = ProgramDatabase(self.config.database)\n\n self.evaluator = Evaluator(\n self.config.evaluator,\n evaluation_file,\n self.llm_evaluator_ensemble,\n self.evaluator_prompt_sampler,\n database=self.database,\n )\n\n logger.info(f\"Initialized EvolveAgent with {initial_program_path} \" f\"and {evaluation_file}\")\n\n def _setup_logging(self) -> None:\n \"\"\"Set up logging\"\"\"\n log_dir = self.config.log_dir or os.path.join(self.output_dir, \"logs\")\n os.makedirs(log_dir, exist_ok=True)\n\n # Set up root logger\n root_logger = logging.getLogger()\n root_logger.setLevel(getattr(logging, self.config.log_level))\n\n # Add file handler\n log_file = os.path.join(log_dir, f\"evolve_agent_{time.strftime('%Y%m%d_%H%M%S')}.log\")\n file_handler = logging.FileHandler(log_file)\n file_handler.setFormatter(\n logging.Formatter(\"%(asctime)s - %(name)s - %(levelname)s - %(message)s\")\n )\n root_logger.addHandler(file_handler)\n\n # Add console handler\n console_handler = logging.StreamHandler()\n console_handler.setFormatter(logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\"))\n root_logger.addHandler(console_handler)\n\n logger.info(f\"Logging to {log_file}\")\n\n async def _generate_new_proposal(\n self,\n parent_proposal: List[str],\n parent_program: str,\n parent_metrics: Dict[str, Any],\n inspirations: List[Program],\n evolution_round: int,\n ) -> List[str]:\n \"\"\"\n Generate a new research proposal based on parent proposal, program, and metrics\n \n Args:\n parent_proposal: The parent program's proposal\n parent_program: The parent program's code\n parent_metrics: The parent program's metrics\n inspirations: List of inspiration programs\n evolution_round: Current evolution round\n \n Returns:\n List of strings representing the new proposal\n \"\"\"\n # Build a prompt for proposal generation\n proposal_prompt = self._build_proposal_prompt(\n parent_proposal=parent_proposal,\n parent_program=parent_program,\n parent_metrics=parent_metrics,\n inspirations=inspirations,\n evolution_round=evolution_round,\n )\n \n # Generate new proposal using LLM\n try:\n proposal_response = await self.llm_ensemble.generate_with_context(\n system_message=proposal_prompt[\"system\"],\n messages=[{\"role\": \"user\", \"content\": proposal_prompt[\"user\"]}],\n )\n \n # Parse the proposal response\n new_proposal = self._parse_proposal_response(proposal_response)\n \n logger.info(f\"Generated new proposal for evolution round {evolution_round}\")\n return new_proposal\n \n except Exception as e:\n logger.warning(f\"Failed to generate new proposal: {e}\")\n # Fallback to parent proposal with some modification\n return self._modify_parent_proposal(parent_proposal, parent_metrics)\n\n def _build_proposal_prompt(\n self,\n parent_proposal: List[str],\n parent_program: str,\n parent_metrics: Dict[str, Any],\n inspirations: List[Program],\n evolution_round: int,\n ) -> Dict[str, str]:\n \"\"\"\n Build a prompt for generating a new research proposal\n \n Args:\n parent_proposal: The parent program's proposal\n parent_program: The parent program's code\n parent_metrics: The parent program's metrics\n inspirations: List of inspiration programs\n evolution_round: Current evolution round\n \n Returns:\n Dictionary with 'system' and 'user' keys\n \"\"\"\n # Format parent proposal\n parent_proposal_str = \"\\n\".join(parent_proposal) if parent_proposal else \"\"\n \n # Format metrics\n metrics_str = _format_metrics(parent_metrics)\n \n # Format inspirations\n inspirations_str = \"\"\n if inspirations:\n inspirations_str = \"\\n\".join([\n f\"Program {i+1}:\\nProposal: {' '.join(prog.proposal if prog.proposal else [])}\\nMetrics: {_format_metrics(prog.metrics)}\"\n for i, prog in enumerate(inspirations[:3])\n ])\n \n system_message = \"\"\"You are a research advisor tasked with evolving and improving research proposals. \nYour goal is to generate a new research proposal that builds upon the current proposal while addressing its limitations and incorporating insights from successful approaches.\n\nFocus on:\n1. Identifying weaknesses in the current approach based on performance metrics\n2. Proposing novel improvements that could enhance performance\n3. Learning from successful inspirations while maintaining originality\n4. Ensuring the new proposal is technically sound and implementable\"\"\"\n\n user_message = f\"\"\"Based on the following information, generate an improved research proposal:\n\n- Current Proposal:\n{parent_proposal_str}\n\n- Current Program:\n```python\n{parent_program}\n```\n\n- Current Metrics\n{metrics_str}\n\nPlease generate a new research proposal that:\n1. Addresses the limitations shown in the current metrics\n2. Incorporates insights from successful approaches\n3. Proposes specific technical improvements\n4. Maintains clarity and technical rigor\n\nReturn the proposal as a clear, concise research abstract.\"\"\"\n\n return {\n \"system\": system_message,\n \"user\": user_message\n }\n\n def _parse_proposal_response(self, response: str) -> List[str]:\n \"\"\"\n Parse the LLM response to extract the new proposal\n \n Args:\n response: Raw LLM response\n \n Returns:\n List of strings representing the proposal\n \"\"\"\n # Clean up the response\n proposal_text = response.strip()\n \n # Remove any markdown formatting\n proposal_text = re.sub(r'^```.*?\\n', '', proposal_text, flags=re.MULTILINE)\n proposal_text = re.sub(r'\\n```$', '', proposal_text, flags=re.MULTILINE)\n \n return [proposal_text]\n\n def _modify_parent_proposal(self, parent_proposal: List[str], parent_metrics: Dict[str, Any]) -> List[str]:\n \"\"\"\n Fallback method to modify parent proposal when generation fails\n \n Args:\n parent_proposal: The parent proposal\n parent_metrics: The parent metrics\n \n Returns:\n Modified proposal\n \"\"\"\n if not parent_proposal:\n return [\"Enhanced research approach with improved methodology and performance optimization.\"]\n \n # Simple modification by adding improvement context\n modified_proposal = []\n for part in parent_proposal:\n modified_part = part\n if \"improvement\" not in part.lower():\n modified_part += \" This approach has been enhanced based on performance analysis and optimization insights.\"\n modified_proposal.append(modified_part)\n \n return modified_proposal\n\n def _load_initial_program(self) -> str:\n \"\"\"Load the initial program from file\"\"\"\n with open(self.initial_program_path, \"r\") as f:\n return f.read()\n\n def _load_initial_proposal(self) -> List[str]:\n \"\"\"Load the initial proposal from file\"\"\"\n with open(self.initial_proposal_path, \"r\") as f:\n proposal = f.read()\n return [proposal]\n\n async def run(\n self,\n iterations: Optional[int] = None,\n target_score: Optional[float] = None,\n ) -> Program:\n \"\"\"\n Run the evolution process\n\n Args:\n iterations: Maximum number of iterations (uses config if None)\n target_score: Target score to reach (continues until reached if specified)\n\n Returns:\n Best program found\n \"\"\"\n max_iterations = iterations or self.config.max_iterations\n\n # Define start_iteration before creating the initial program\n start_iteration = self.database.last_iteration\n\n # Only add initial program if starting fresh (not resuming from checkpoint)\n # Check if we're resuming AND no program matches initial code to avoid pollution\n should_add_initial = (\n start_iteration == 0\n and len(self.database.programs) == 0\n and not any(\n p.code == self.initial_program_code for p in self.database.programs.values()\n )\n and self.initial_proposal\n )\n\n if should_add_initial:\n logger.info(\"Adding initial program to database\")\n initial_program_id = str(uuid.uuid4())\n\n # Evaluate the initial program\n initial_metrics = await self.evaluator.evaluate_program(\n self.initial_program_code, initial_program_id\n )\n\n initial_idea_reward = await self.reward_model.score_research_proposal(self.initial_proposal)\n\n initial_program = Program(\n id=initial_program_id,\n code=self.initial_program_code,\n language=self.language,\n metrics=initial_metrics,\n proposal=self.initial_proposal,\n idea_reward=initial_idea_reward,\n iteration_found=start_iteration,\n )\n\n self.database.add(initial_program)\n else:\n logger.info(\n f\"Skipping initial program addition (resuming from iteration {start_iteration} with {len(self.database.programs)} existing programs)\"\n )\n\n # Main evolution loop\n total_iterations = start_iteration + max_iterations\n\n logger.info(\n f\"Starting evolution from iteration {start_iteration} for {max_iterations} iterations (total: {total_iterations})\"\n )\n\n # Island-based evolution variables\n programs_per_island = max(\n 1, max_iterations // (self.config.database.num_islands * 10)\n ) # Dynamic allocation\n current_island_counter = 0\n\n logger.info(f\"Using island-based evolution with {self.config.database.num_islands} islands\")\n self.database.log_island_status()\n\n for i in range(start_iteration, total_iterations):\n iteration_start = time.time()\n\n # Manage island evolution - switch islands periodically\n if i > start_iteration and current_island_counter >= programs_per_island:\n self.database.next_island()\n current_island_counter = 0\n logger.debug(f\"Switched to island {self.database.current_island}\")\n\n current_island_counter += 1\n\n # Sample parent and inspirations from current island\n parent, inspirations = self.database.sample()\n\n # Get artifacts for the parent program if available\n parent_artifacts = self.database.get_artifacts(parent.id)\n\n # Step 1: Generate new proposal using parent's proposal, program, and metrics\n new_proposal = await self._generate_new_proposal(\n parent_proposal=parent.proposal,\n parent_program=parent.code,\n parent_metrics=parent.metrics,\n inspirations=inspirations,\n evolution_round=i\n )\n \n # Step 2: Score the new proposal using RewardModel\n new_proposal_results = await self.reward_model.score_research_proposal(new_proposal)\n new_proposal_score = new_proposal_results[0].get('score', -1.0)\n\n # Step 3: Check if proposal score meets threshold\n if new_proposal_score < self.config.rewardmodel.proposal_score_threshold:\n logger.info(f\"Iteration {i+1}: Proposal score {new_proposal_score:.4f} below threshold \"\n f\"{self.config.rewardmodel.proposal_score_threshold:.4f}, skipping program generation\")\n continue\n \n logger.info(f\"Iteration {i+1}: Proposal score: {new_proposal_score:.4f} / 10\")\n \n\n # Step 4: Build prompt for program generation using all information\n prompt = self.prompt_sampler.build_prompt(\n current_program=parent.code,\n parent_program=parent.code,\n program_metrics=parent.metrics,\n previous_programs=[p.to_dict() for p in self.database.get_top_programs(3)],\n top_programs=[p.to_dict() for p in inspirations],\n language=self.language,\n evolution_round=i,\n allow_full_rewrite=self.config.allow_full_rewrites,\n program_artifacts=parent_artifacts if parent_artifacts else None,\n current_proposal=new_proposal,\n parent_proposal=parent.proposal,\n proposal_score=new_proposal_score,\n )\n\n # Generate code modification\n try:\n llm_response = await self.llm_ensemble.generate_with_context(\n system_message=prompt[\"system\"],\n messages=[{\"role\": \"user\", \"content\": prompt[\"user\"]}],\n )\n\n # Parse the response\n if self.config.diff_based_evolution:\n diff_blocks = extract_diffs(llm_response)\n if not diff_blocks:\n logger.warning(f\"Iteration {i+1}: No valid diffs found in response\")\n continue\n # Apply the diffs\n child_code = apply_diff(parent.code, llm_response)\n changes_summary = format_diff_summary(diff_blocks)\n\n logger.info(f\"Diff is applied successfully! \")\n else:\n # Parse full rewrite\n new_code = parse_full_rewrite(llm_response, self.language)\n\n if not new_code:\n logger.warning(f\"Iteration {i+1}: No valid code found in response\")\n continue\n\n child_code = new_code\n changes_summary = \"Full rewrite\"\n\n # Check code length\n if len(child_code) > self.config.max_code_length:\n logger.warning(\n f\"Iteration {i+1}: Generated code exceeds maximum length \"\n f\"({len(child_code)} > {self.config.max_code_length})\"\n )\n continue\n\n # Evaluate the child program\n child_id = str(uuid.uuid4())\n child_metrics = await self.evaluator.evaluate_program(child_code, child_id)\n\n # Handle artifacts if they exist\n artifacts = self.evaluator.get_pending_artifacts(child_id)\n\n # Create a child program with the new proposal\n child_program = Program(\n id=child_id,\n code=child_code,\n language=self.language,\n parent_id=parent.id,\n generation=parent.generation + 1,\n metrics=child_metrics,\n proposal=new_proposal,\n idea_reward=new_proposal_score,\n metadata={\n \"changes\": changes_summary,\n \"parent_metrics\": parent.metrics,\n },\n )\n\n # Add to database (will be added to current island)\n self.database.add(child_program, iteration=i + 1)\n\n # Log prompts\n self.database.log_prompt(\n template_key=(\n \"full_rewrite_user\" if self.config.allow_full_rewrites else \"diff_user\"\n ),\n program_id=child_id,\n prompt=prompt,\n responses=[llm_response],\n )\n\n # Store artifacts if they exist\n if artifacts:\n self.database.store_artifacts(child_id, artifacts)\n\n # Log prompts\n self.database.log_prompt(\n template_key=(\n \"full_rewrite_user\" if self.config.allow_full_rewrites else \"diff_user\"\n ),\n program_id=child_id,\n prompt=prompt,\n responses=[llm_response],\n )\n\n # Increment generation for current island\n self.database.increment_island_generation()\n\n # Check if migration should occur\n if self.database.should_migrate():\n logger.info(f\"Performing migration at iteration {i+1}\")\n self.database.migrate_programs()\n self.database.log_island_status()\n\n # Log progress\n iteration_time = time.time() - iteration_start\n self._log_iteration(i, parent, child_program, iteration_time)\n\n # Specifically check if this is the new best program\n if self.database.best_program_id == child_program.id:\n logger.info(f\"🌟 New best solution found at iteration {i+1}: {child_program.id}\")\n logger.info(f\"Metrics: {format_metrics_safe(child_program.metrics)}\")\n\n # Save checkpoint\n if (i + 1) % self.config.checkpoint_interval == 0:\n self._save_checkpoint(i + 1)\n # Also log island status at checkpoints\n logger.info(f\"Island status at checkpoint {i+1}:\")\n self.database.log_island_status()\n\n # Check if target score reached\n if target_score is not None:\n # Only consider numeric metrics for target score calculation\n numeric_metrics = [\n v\n for v in child_metrics.values()\n if isinstance(v, (int, float)) and not isinstance(v, bool)\n ]\n if numeric_metrics:\n avg_score = sum(numeric_metrics) / len(numeric_metrics)\n if avg_score >= target_score:\n logger.info(\n f\"Target score {target_score} reached after {i+1} iterations\"\n )\n break\n\n except Exception as e:\n logger.exception(f\"Error in iteration {i+1}: {str(e)}\")\n continue\n\n # Get the best program using our tracking mechanism\n best_program = None\n if self.database.best_program_id:\n best_program = self.database.get(self.database.best_program_id)\n logger.info(f\"Using tracked best program: {self.database.best_program_id}\")\n\n # Fallback to calculating best program if tracked program not found\n if best_program is None:\n best_program = self.database.get_best_program()\n logger.info(\"Using calculated best program (tracked program not found)\")\n\n # Check if there's a better program by combined_score that wasn't tracked\n if \"combined_score\" in best_program.metrics:\n best_by_combined = self.database.get_best_program(metric=\"combined_score\")\n if (\n best_by_combined\n and best_by_combined.id != best_program.id\n and \"combined_score\" in best_by_combined.metrics\n ):\n # If the combined_score of this program is significantly better, use it instead\n if (\n best_by_combined.metrics[\"combined_score\"]\n > best_program.metrics[\"combined_score\"] + 0.02\n ):\n logger.warning(\n f\"Found program with better combined_score: {best_by_combined.id}\"\n )\n logger.warning(\n f\"Score difference: {best_program.metrics['combined_score']:.4f} vs {best_by_combined.metrics['combined_score']:.4f}\"\n )\n best_program = best_by_combined\n\n if best_program:\n logger.info(\n f\"Evolution complete. Best program has metrics: \"\n f\"{format_metrics_safe(best_program.metrics)}\"\n )\n\n # Save the best program (using our tracked best program)\n self._save_best_program(best_program)\n\n return best_program\n else:\n logger.warning(\"No valid programs found during evolution\")\n # Return None if no programs found instead of undefined initial_program\n return None\n\n def _log_iteration(\n self,\n iteration: int,\n parent: Program,\n child: Program,\n elapsed_time: float,\n ) -> None:\n \"\"\"\n Log iteration progress\n\n Args:\n iteration: Iteration number\n parent: Parent program\n child: Child program\n elapsed_time: Elapsed time in seconds\n \"\"\"\n # Calculate improvement using safe formatting\n improvement_str = format_improvement_safe(parent.metrics, child.metrics)\n\n logger.info(\n f\"Iteration {iteration+1}: Child {child.id} from parent {parent.id} \"\n f\"in {elapsed_time:.2f}s. Metrics: \"\n f\"{format_metrics_safe(child.metrics)} \"\n f\"(Δ: {improvement_str})\"\n )\n\n def _save_checkpoint(self, iteration: int) -> None:\n \"\"\"\n Save a checkpoint\n\n Args:\n iteration: Current iteration number\n \"\"\"\n checkpoint_dir = os.path.join(self.output_dir, \"checkpoints\")\n os.makedirs(checkpoint_dir, exist_ok=True)\n\n # Create specific checkpoint directory\n checkpoint_path = os.path.join(checkpoint_dir, f\"checkpoint_{iteration}\")\n os.makedirs(checkpoint_path, exist_ok=True)\n\n # Save the database\n self.database.save(checkpoint_path, iteration)\n\n # Save the best program found so far\n best_program = None\n if self.database.best_program_id:\n best_program = self.database.get(self.database.best_program_id)\n else:\n best_program = self.database.get_best_program()\n\n if best_program:\n # Save the best program at this checkpoint\n best_program_path = os.path.join(checkpoint_path, f\"best_program{self.file_extension}\")\n with open(best_program_path, \"w\") as f:\n f.write(best_program.code)\n\n # Save metrics\n best_program_info_path = os.path.join(checkpoint_path, \"best_program_info.json\")\n with open(best_program_info_path, \"w\") as f:\n import json\n\n json.dump(\n {\n \"id\": best_program.id,\n \"generation\": best_program.generation,\n \"iteration\": best_program.iteration_found,\n \"current_iteration\": iteration,\n \"metrics\": best_program.metrics,\n \"language\": best_program.language,\n \"timestamp\": best_program.timestamp,\n \"saved_at\": time.time(),\n },\n f,\n indent=2,\n )\n\n logger.info(\n f\"Saved best program at checkpoint {iteration} with metrics: \"\n f\"{format_metrics_safe(best_program.metrics)}\"\n )\n\n logger.info(f\"Saved checkpoint at iteration {iteration} to {checkpoint_path}\")\n\n def _save_best_program(self, program: Optional[Program] = None) -> None:\n \"\"\"\n Save the best program\n\n Args:\n program: Best program (if None, uses the tracked best program)\n \"\"\"\n # If no program is provided, use the tracked best program from the database\n if program is None:\n if self.database.best_program_id:\n program = self.database.get(self.database.best_program_id)\n else:\n # Fallback to calculating best program if no tracked best program\n program = self.database.get_best_program()\n\n if not program:\n logger.warning(\"No best program found to save\")\n return\n\n best_dir = os.path.join(self.output_dir, \"best\")\n os.makedirs(best_dir, exist_ok=True)\n\n # Use the extension from the initial program file\n filename = f\"best_program{self.file_extension}\"\n code_path = os.path.join(best_dir, filename)\n\n with open(code_path, \"w\") as f:\n f.write(program.code)\n\n # Save complete program info including metrics\n info_path = os.path.join(best_dir, \"best_program_info.json\")\n with open(info_path, \"w\") as f:\n import json\n\n json.dump(\n {\n \"id\": program.id,\n \"generation\": program.generation,\n \"iteration\": program.iteration_found,\n \"timestamp\": program.timestamp,\n \"parent_id\": program.parent_id,\n \"metrics\": program.metrics,\n \"language\": program.language,\n \"saved_at\": time.time(),\n },\n f,\n indent=2,\n )\n\n logger.info(f\"Saved best program to {code_path} with program info to {info_path}\")\n\nif __name__=='__main__':\n # Initialize the system\n evolve_agent = EvolveAgent(\n initial_program_path=\"/data/zhuotaodeng/yzj/alpha-research/results/initial_program.py\",\n evaluation_file=\"/data/zhuotaodeng/yzj/alpha-research/results/evaluator.py\",\n config_path=\"/data/zhuotaodeng/yzj/alpha-research/configs/default_config.yaml\"\n )\n\n print(evolve_agent)\n # # Run the evolution\n # best_program = await evolve.run(iterations=1000)\n # print(f\"Best program metrics:\")\n # for name, value in best_program.metrics.items():\n # print(f\" {name}: {value:.4f}\")"}}, "reward_model/train/preprocess.py": {"8e6e0d882091ce091e7f72f0bf55f4bf8e5a6f66": {"t": 1785599217.2059565, "n": 6411, "h": "2a604543424cc521", "c": "from dataclasses import dataclass, field\nimport logging\nimport pathlib\nimport copy\nfrom typing import Dict, Optional, Sequence\nimport torch\nfrom torch.utils.data import Dataset\n\nimport transformers\nfrom transformers import AutoTokenizer\nfrom datasets import load_dataset\n\nIGNORE_INDEX = -100\nDEFAULT_PAD_TOKEN = \"[PAD]\"\nDEFAULT_EOS_TOKEN = \"</s>\"\nDEFAULT_BOS_TOKEN = \"<s>\"\nDEFAULT_UNK_TOKEN = \"<unk>\"\n_MAGIC_SPLITTER_ = \"-[[]]-this-is-really-our-highest-priority-[[]]-\"\nSYSTEM_PROMPT = \"You are an expert reviewer tasked with evaluating the quality of a research proposal. \"\n\n\nclass SupervisedDataset(Dataset):\n \"\"\"Dataset for supervised fine-tuning.\"\"\"\n\n def __init__(self, data_path: str, tokenizer: transformers.PreTrainedTokenizer):\n super(SupervisedDataset, self).__init__()\n logging.warning(\"Loading data...\")\n list_data_dict = load_dataset('json', data_files=data_path, split='train')\n logging.warning(\"Formatting inputs...\")\n sources = [\n (\n prompt_format(tokenizer, example)\n )\n for example in list_data_dict\n ]\n targets = [\n f\"{example['response']}{'<|im_end|>'}\" for example in list_data_dict\n ]\n\n logging.warning(\"Tokenizing inputs... This may take some time...\")\n data_dict = preprocess(sources, targets, tokenizer)\n\n self.input_ids = data_dict[\"input_ids\"]\n self.labels = data_dict[\"labels\"]\n\n def __len__(self):\n return len(self.input_ids)\n\n def __getitem__(self, i) -> Dict[str, torch.Tensor]:\n return dict(input_ids=self.input_ids[i], labels=self.labels[i])\n \n\ndef save_dataset(dataset: SupervisedDataset, save_path: str):\n\n save_dir = pathlib.Path(save_path)\n save_dir.mkdir(parents=True, exist_ok=True)\n\n torch.save(dataset.input_ids, save_dir / \"input_ids.pt\")\n torch.save(dataset.labels, save_dir / \"labels.pt\")\n logging.info(f\"Dataset saved to {save_dir}\")\n\n\ndef load_from_pt(save_path: str) -> SupervisedDataset:\n\n save_dir = pathlib.Path(save_path)\n\n # Load input_ids and labels\n input_ids = torch.load(save_dir / \"input_ids.pt\")\n labels = torch.load(save_dir / \"labels.pt\")\n\n # Create an empty SupervisedDataset instance\n dataset = SupervisedDataset.__new__(SupervisedDataset)\n dataset.input_ids = input_ids\n dataset.labels = labels\n\n logging.info(f\"Dataset loaded from {save_dir}\")\n return dataset\n\n\ndef _tokenize_fn(\n strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer\n) -> Dict:\n \"\"\"Tokenize a list of strings.\"\"\"\n tokenized_list = [\n tokenizer(\n text,\n return_tensors=\"pt\",\n padding=\"longest\",\n max_length=tokenizer.model_max_length,\n truncation=True,\n )\n for text in strings\n ]\n input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list]\n input_ids_lens = labels_lens = [\n tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item()\n for tokenized in tokenized_list\n ]\n return dict(\n input_ids=input_ids,\n labels=labels,\n input_ids_lens=input_ids_lens,\n labels_lens=labels_lens,\n )\n\ndef preprocess(\n sources: Sequence[str],\n targets: Sequence[str],\n tokenizer: transformers.PreTrainedTokenizer,\n) -> Dict:\n \"\"\"Preprocess the data by tokenizing.\"\"\"\n examples = [s + t for s, t in zip(sources, targets)]\n examples_tokenized, sources_tokenized = [\n _tokenize_fn(strings, tokenizer) for strings in (examples, sources)\n ]\n input_ids = examples_tokenized[\"input_ids\"]\n labels = copy.deepcopy(input_ids)\n for label, source_len in zip(labels, sources_tokenized[\"input_ids_lens\"]):\n label[:source_len] = IGNORE_INDEX\n return dict(input_ids=input_ids, labels=labels)\n\n\n@dataclass\nclass DataCollatorForSupervisedDataset(object):\n \"\"\"Collate examples for supervised fine-tuning.\"\"\"\n\n tokenizer: transformers.PreTrainedTokenizer\n\n def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:\n input_ids, labels = tuple(\n [instance[key] for instance in instances] for key in (\"input_ids\", \"labels\")\n )\n input_ids = torch.nn.utils.rnn.pad_sequence(\n input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id\n )\n labels = torch.nn.utils.rnn.pad_sequence(\n labels, batch_first=True, padding_value=IGNORE_INDEX\n )\n return dict(\n input_ids=input_ids,\n labels=labels,\n attention_mask=input_ids.ne(self.tokenizer.pad_token_id),\n )\n\n\ndef make_supervised_data_module(\n tokenizer: transformers.PreTrainedTokenizer, data_path\n) -> Dict:\n \"\"\"Make dataset and collator for supervised fine-tuning.\"\"\"\n train_dataset = SupervisedDataset(\n tokenizer=tokenizer, data_path=data_path\n )\n data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)\n print(f\"len={len(train_dataset)}\")\n return dict(\n train_dataset=train_dataset, eval_dataset=None, data_collator=data_collator\n )\n\n\ndef prompt_format(tokenizer, example):\n question = example['question'].strip()\n messages = [\n {\n \"role\": \"system\",\n \"content\": SYSTEM_PROMPT\n }, \n {\n \"role\": \"user\",\n \"content\": question\n },\n {\n \"role\": \"assistant\",\n \"content\": '' + _MAGIC_SPLITTER_ \n }\n ]\n return tokenizer.apply_chat_template(messages, tokenize=False).split(_MAGIC_SPLITTER_)[0]\n\nif __name__ == '__main__':\n tokenizer = AutoTokenizer.from_pretrained('/work/zhuotaodeng/yzj/pretrained_models_ms/Qwen/Qwen2___5-7B-Instruct')\n # data_path = '/data/zhuotaodeng/test-time-scaling/z1/data/openthought_evol-221k.json'\n # ds = make_supervised_data_module(tokenizer, data_path)\n # save_dataset(ds['train_dataset'],'/data/zhuotaodeng/test-time-scaling/z1/data/qwen')\n\n ds = load_dataset('/data/zhuotaodeng/test-time-scaling/z1/data/qwen')\n data = ds[1]\n decoded_input = tokenizer.decode(data['input_ids'], skip_special_tokens=True)\n print(\"Decoded input_ids:\", decoded_input)\n filtered_labels = data['labels'][data['labels'] != -100]\n decoded_labels = tokenizer.decode(filtered_labels, skip_special_tokens=True)\n print(\"Decoded labels:\", decoded_labels)\n"}}}} |