{"repo": "0xnu/llm-benchmark", "files": {"scripts/benchmark.py": {"0a1da09e3f670e78794b668f37e222ad1c521216": {"t": 1785674097.1959555, "n": 24841, "h": "212e4ce0b17045ff", "c": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\n\"\"\"\n@name: benchmark.py\n@author: Finbarrs Oketunji\n@contact: f@finbarrs.eu\n@time: Wednesday July 09 05:05:15 2025\n@updated: Tuesday July 15 18:02:00 2025\n@desc: Test and compare different large language models on various tasks\n@run: python3 benchmark.py\n\"\"\"\n\nimport asyncio\nimport json\nimport os\nfrom typing import Dict, List, Any\nfrom dataclasses import asdict\nfrom datetime import datetime\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom scripts.utils import (\n    BenchmarkResult, BenchmarkTask, QualityEvaluator, APIKeyManager, setup_environment\n)\nfrom scripts.models import (\n    LLMInterface, OpenAIInterface, AnthropicInterface, DeepSeekInterface,\n    XAIInterface, GeminiInterface, MoonshotInterface, QwenInterface, MistralInterface\n)\n\n\nclass LLMBenchmark:\n    def __init__(self, results_dir: str = \"results\"):\n        self.models: List[LLMInterface] = []\n        self.tasks: List[BenchmarkTask] = []\n        self.results: List[BenchmarkResult] = []\n        self.results_dir = results_dir\n\n        # Create results directory if it doesn't exist\n        os.makedirs(self.results_dir, exist_ok=True)\n\n    def add_model(self, model: LLMInterface):\n        \"\"\"Add a model to benchmark\"\"\"\n        self.models.append(model)\n\n    def add_task(self, task: BenchmarkTask):\n        \"\"\"Add a benchmark task\"\"\"\n        self.tasks.append(task)\n\n    def create_standard_tasks(self):\n        \"\"\"Create standard benchmark tasks\"\"\"\n        tasks = [\n            BenchmarkTask(\n                name=\"Code Generation\",\n                prompt=\"Write a Python function that implements binary search on a sorted list. Include proper error handling and documentation.\",\n                category=\"Programming\",\n                expected_length=\"medium\",\n                evaluation_criteria=[\"correctness\", \"efficiency\", \"documentation\"]\n            ),\n            BenchmarkTask(\n                name=\"Mathematical Reasoning\",\n                prompt=\"Solve this step by step: If a train travels 120 miles in 2 hours, and then increases its speed by 25% for the next 90 miles, what is the total travel time?\",\n                category=\"Mathematics\",\n                expected_length=\"medium\",\n                evaluation_criteria=[\"accuracy\", \"reasoning\", \"clarity\"]\n            ),\n            BenchmarkTask(\n                name=\"Creative Writing\",\n                prompt=\"Write a short story (200 words) about a robot who discovers emotions. Focus on the moment of discovery.\",\n                category=\"Creative\",\n                expected_length=\"medium\",\n                evaluation_criteria=[\"creativity\", \"coherence\", \"emotional depth\"]\n            ),\n            BenchmarkTask(\n                name=\"Data Analysis\",\n                prompt=\"Explain the key differences between correlation and causation. Provide practical examples and explain why this distinction matters in data science.\",\n                category=\"Analysis\",\n                expected_length=\"long\",\n                evaluation_criteria=[\"accuracy\", \"examples\", \"practical relevance\"]\n            ),\n            BenchmarkTask(\n                name=\"Logical Reasoning\",\n                prompt=\"All roses are flowers. Some flowers are red. Some red things are beautiful. Can we conclude that some roses are beautiful? Explain your reasoning.\",\n                category=\"Logic\",\n                expected_length=\"short\",\n                evaluation_criteria=[\"logical validity\", \"clarity\", \"completeness\"]\n            ),\n            BenchmarkTask(\n                name=\"Summarisation\",\n                prompt=\"Summarise the key principles of effective communication in professional settings. Focus on practical, actionable advice.\",\n                category=\"Communication\",\n                expected_length=\"medium\",\n                evaluation_criteria=[\"completeness\", \"practicality\", \"organisation\"]\n            ),\n            BenchmarkTask(\n                name=\"Technical Explanation\",\n                prompt=\"Explain how neural networks learn through backpropagation. Make it accessible to someone with basic programming knowledge but no machine learning background.\",\n                category=\"Education\",\n                expected_length=\"long\",\n                evaluation_criteria=[\"clarity\", \"accuracy\", \"accessibility\"]\n            ),\n            BenchmarkTask(\n                name=\"Problem Solving\",\n                prompt=\"You have 8 balls that look identical. One ball is slightly heavier than the others. Using a balance scale only twice, how can you identify the heavier ball?\",\n                category=\"Logic\",\n                expected_length=\"medium\",\n                evaluation_criteria=[\"logical approach\", \"efficiency\", \"clarity\"]\n            )\n        ]\n\n        for task in tasks:\n            self.add_task(task)\n\n    async def run_benchmark(self, iterations: int = 3) -> List[BenchmarkResult]:\n        \"\"\"Run benchmark across all models and tasks\"\"\"\n        print(f\"Running benchmark with {len(self.models)} models and {len(self.tasks)} tasks\")\n        print(f\"Iterations per task: {iterations}\")\n\n        for model in self.models:\n            print(f\"\\nTesting {model.get_model_name()}...\")\n\n            for task in self.tasks:\n                print(f\"  Task: {task.name}\")\n\n                # Run multiple iterations for each task\n                for i in range(iterations):\n                    response, metadata = await model.generate(task.prompt)\n\n                    # Calculate quality score\n                    quality_score = QualityEvaluator.evaluate_response(task, response)\n\n                    result = BenchmarkResult(\n                        model_name=model.get_model_name(),\n                        task_name=task.name,\n                        response=response,\n                        latency=metadata[\"latency\"],\n                        input_tokens=int(metadata[\"input_tokens\"]),\n                        output_tokens=int(metadata[\"output_tokens\"]),\n                        cost_estimate=model.estimate_cost(\n                            int(metadata[\"input_tokens\"]),\n                            int(metadata[\"output_tokens\"])\n                        ),\n                        timestamp=datetime.now(),\n                        quality_score=quality_score,\n                        error=metadata.get(\"error\")\n                    )\n\n                    self.results.append(result)\n\n                    # Brief pause between requests\n                    await asyncio.sleep(0.1)\n\n        return self.results\n\n    def analyse_results(self) -> Dict[str, Any]:\n        \"\"\"Analyse benchmark results\"\"\"\n        if not self.results:\n            return {\"error\": \"No results to analyse\"}\n\n        # Convert results to DataFrame for easier analysis\n        df = pd.DataFrame([asdict(r) for r in self.results])\n\n        # Calculate metrics by model\n        model_metrics = {}\n\n        for model in df['model_name'].unique():\n            model_data = df[df['model_name'] == model]\n\n            model_metrics[model] = {\n                'avg_latency': model_data['latency'].mean(),\n                'median_latency': model_data['latency'].median(),\n                'avg_cost': model_data['cost_estimate'].mean(),\n                'total_tokens': model_data['input_tokens'].sum() + model_data['output_tokens'].sum(),\n                'error_rate': (model_data['error'].notna().sum() / len(model_data)) * 100,\n                'avg_response_length': model_data['response'].str.len().mean(),\n                'avg_quality_score': model_data['quality_score'].mean(),\n                'quality_score_std': model_data['quality_score'].std(),\n                'min_quality_score': model_data['quality_score'].min(),\n                'max_quality_score': model_data['quality_score'].max()\n            }\n\n        # Task-specific analysis\n        task_metrics = {}\n        for task in df['task_name'].unique():\n            task_data = df[df['task_name'] == task]\n\n            task_metrics[task] = {\n                'models_tested': task_data['model_name'].nunique(),\n                'avg_latency_across_models': task_data['latency'].mean(),\n                'response_length_variance': task_data['response'].str.len().var(),\n                'avg_quality_score_across_models': task_data['quality_score'].mean(),\n                'quality_score_range': task_data['quality_score'].max() - task_data['quality_score'].min(),\n                'best_performing_model': task_data.loc[task_data['quality_score'].idxmax(), 'model_name'] if len(task_data) > 0 else None\n            }\n\n        return {\n            'model_metrics': model_metrics,\n            'task_metrics': task_metrics,\n            'overall_stats': {\n                'total_requests': len(df),\n                'total_cost': df['cost_estimate'].sum(),\n                'avg_latency': df['latency'].mean(),\n                'overall_avg_quality_score': df['quality_score'].mean(),\n                'quality_score_std': df['quality_score'].std(),\n                'best_overall_model': df.groupby('model_name')['quality_score'].mean().idxmax() if len(df) > 0 else None\n            }\n        }\n\n    def generate_report(self, output_file: str = None):\n        \"\"\"Generate detailed benchmark report\"\"\"\n        if output_file is None:\n            timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n            output_file = os.path.join(self.results_dir, f\"benchmark_report_{timestamp}.json\")\n\n        analysis = self.analyse_results()\n\n        report = {\n            'benchmark_info': {\n                'timestamp': datetime.now().isoformat(),\n                'models_tested': len(self.models),\n                'tasks_completed': len(self.tasks),\n                'total_requests': len(self.results)\n            },\n            'results': analysis,\n            'raw_data': [asdict(r) for r in self.results]\n        }\n\n        with open(output_file, 'w') as f:\n            json.dump(report, f, indent=2, default=str)\n\n        print(f\"Report saved to {output_file}\")\n        return report\n\n    def create_visualisations(self):\n        \"\"\"Create visualisations of benchmark results\"\"\"\n        if not self.results:\n            print(\"No results to visualise\")\n            return\n\n        df = pd.DataFrame([asdict(r) for r in self.results])\n\n        # Set up the plotting style\n        plt.style.use('seaborn-v0_8')\n        fig, axes = plt.subplots(2, 3, figsize=(20, 12))\n\n        # 1. Quality Score comparison\n        quality_by_model = df.groupby('model_name')['quality_score'].mean()\n        axes[0, 0].bar(quality_by_model.index, quality_by_model.values, color='lightblue')\n        axes[0, 0].set_title('Average Quality Score by Model')\n        axes[0, 0].set_ylabel('Quality Score (%)')\n        axes[0, 0].set_ylim(0, 100)\n        axes[0, 0].tick_params(axis='x', rotation=45)\n\n        # Add score labels on bars\n        for i, v in enumerate(quality_by_model.values):\n            axes[0, 0].text(i, v + 1, f'{v:.1f}%', ha='center', va='bottom')\n\n        # 2. Latency comparison\n        axes[0, 1].boxplot([df[df['model_name'] == model]['latency'].values\n                           for model in df['model_name'].unique()],\n                          labels=df['model_name'].unique())\n        axes[0, 1].set_title('Response Latency by Model')\n        axes[0, 1].set_ylabel('Latency (seconds)')\n        axes[0, 1].tick_params(axis='x', rotation=45)\n\n        # 3. Cost comparison\n        cost_by_model = df.groupby('model_name')['cost_estimate'].mean()\n        axes[0, 2].bar(cost_by_model.index, cost_by_model.values, color='lightcoral')\n        axes[0, 2].set_title('Average Cost per Request by Model')\n        axes[0, 2].set_ylabel('Cost ($)')\n        axes[0, 2].tick_params(axis='x', rotation=45)\n\n        # 4. Task performance heatmap (Quality Scores)\n        pivot_quality = df.pivot_table(values='quality_score',\n                                      index='task_name',\n                                      columns='model_name',\n                                      aggfunc='mean')\n        sns.heatmap(pivot_quality, annot=True, fmt='.1f', ax=axes[1, 0], cmap='RdYlGn', vmin=0, vmax=100)\n        axes[1, 0].set_title('Quality Scores by Task and Model (%)')\n\n        # 5. Quality vs Latency scatter\n        for model in df['model_name'].unique():\n            model_data = df[df['model_name'] == model]\n            axes[1, 1].scatter(model_data['latency'], model_data['quality_score'],\n                             label=model, alpha=0.7, s=50)\n        axes[1, 1].set_xlabel('Latency (seconds)')\n        axes[1, 1].set_ylabel('Quality Score (%)')\n        axes[1, 1].set_title('Quality vs Latency Trade-off')\n        axes[1, 1].legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n        axes[1, 1].grid(True, alpha=0.3)\n\n        # 6. Quality vs Cost scatter\n        for model in df['model_name'].unique():\n            model_data = df[df['model_name'] == model]\n            axes[1, 2].scatter(model_data['cost_estimate'], model_data['quality_score'],\n                             label=model, alpha=0.7, s=50)\n        axes[1, 2].set_xlabel('Cost ($)')\n        axes[1, 2].set_ylabel('Quality Score (%)')\n        axes[1, 2].set_title('Quality vs Cost Trade-off')\n        axes[1, 2].legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n        axes[1, 2].grid(True, alpha=0.3)\n\n        plt.tight_layout()\n\n        # Save to results directory with timestamp\n        timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n        results_chart_path = os.path.join(self.results_dir, f'llm_benchmark_results_{timestamp}.png')\n        plt.savefig(results_chart_path, dpi=300, bbox_inches='tight')\n        plt.show()\n\n        # Create additional detailed analysis charts\n        self._create_detailed_charts(df, timestamp)\n\n    def _create_detailed_charts(self, df, timestamp):\n        \"\"\"Create additional detailed analysis charts\"\"\"\n        fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n\n        # 1. Cost efficiency (quality per dollar)\n        model_efficiency = df.groupby('model_name').apply(\n            lambda x: x['quality_score'].mean() / (x['cost_estimate'].mean() * 1000) if x['cost_estimate'].mean() > 0 else x['quality_score'].mean() * 1000\n        ).sort_values(ascending=False)\n\n        axes[0, 0].bar(model_efficiency.index, model_efficiency.values, color='lightgreen')\n        axes[0, 0].set_title('Cost Efficiency (Quality Score per $0.001)')\n        axes[0, 0].set_ylabel('Quality Score per $0.001')\n        axes[0, 0].tick_params(axis='x', rotation=45)\n\n        # Add efficiency labels\n        for i, v in enumerate(model_efficiency.values):\n            axes[0, 0].text(i, v + 0.5, f'{v:.1f}', ha='center', va='bottom')\n\n        # 2. Speed efficiency (quality per second)\n        speed_efficiency = df.groupby('model_name').apply(\n            lambda x: x['quality_score'].mean() / x['latency'].mean()\n        ).sort_values(ascending=False)\n\n        axes[0, 1].bar(speed_efficiency.index, speed_efficiency.values, color='orange')\n        axes[0, 1].set_title('Speed Efficiency (Quality Score per Second)')\n        axes[0, 1].set_ylabel('Quality Score per Second')\n        axes[0, 1].tick_params(axis='x', rotation=45)\n\n        # 3. Task category performance\n        category_performance = df.merge(\n            pd.DataFrame([(task.name, task.category) for task in self.tasks],\n                        columns=['task_name', 'category']),\n            on='task_name'\n        ).groupby(['model_name', 'category'])['quality_score'].mean().unstack(fill_value=0)\n\n        category_performance.plot(kind='bar', ax=axes[1, 0], width=0.8)\n        axes[1, 0].set_title('Performance by Task Category')\n        axes[1, 0].set_ylabel('Average Quality Score (%)')\n        axes[1, 0].legend(title='Category', bbox_to_anchor=(1.05, 1), loc='upper left')\n        axes[1, 0].tick_params(axis='x', rotation=45)\n\n        # 4. Error rates\n        error_rates = df.groupby('model_name').apply(\n            lambda x: (x['error'].notna().sum() / len(x)) * 100\n        )\n\n        axes[1, 1].bar(error_rates.index, error_rates.values, color='red', alpha=0.7)\n        axes[1, 1].set_title('Error Rate by Model')\n        axes[1, 1].set_ylabel('Error Rate (%)')\n        axes[1, 1].tick_params(axis='x', rotation=45)\n\n        plt.tight_layout()\n\n        # Save detailed charts to results directory\n        detailed_chart_path = os.path.join(self.results_dir, f'llm_benchmark_detailed_{timestamp}.png')\n        plt.savefig(detailed_chart_path, dpi=300, bbox_inches='tight')\n        plt.show()\n\n    def print_summary_table(self):\n        \"\"\"Print a formatted summary table of results\"\"\"\n        if not self.results:\n            print(\"No results to summarise\")\n            return\n\n        df = pd.DataFrame([asdict(r) for r in self.results])\n\n        # Calculate summary metrics\n        summary = df.groupby('model_name').agg({\n            'quality_score': ['mean', 'std'],\n            'latency': ['mean', 'median'],\n            'cost_estimate': ['mean', 'sum'],\n            'error': lambda x: (x.notna().sum() / len(x)) * 100\n        }).round(3)\n\n        summary.columns = ['Avg Quality', 'Quality Std', 'Avg Latency', 'Med Latency', 'Avg Cost', 'Total Cost', 'Error Rate']\n\n        print(\"\\n\" + \"=\"*80)\n        print(\"LLM BENCHMARK SUMMARY\")\n        print(\"=\"*80)\n        print(summary.to_string())\n        print(\"=\"*80)\n\n        # Print best performers in each category\n        best_quality = df.groupby('model_name')['quality_score'].mean().idxmax()\n        best_speed = df.groupby('model_name')['latency'].mean().idxmin()\n        best_cost = df.groupby('model_name')['cost_estimate'].mean().idxmin()\n\n        print(f\"\\nBest Overall Quality: {best_quality}\")\n        print(f\"Fastest Response: {best_speed}\")\n        print(f\"Most Cost-Effective: {best_cost}\")\n\n    def save_raw_data(self, filename: str = None):\n        \"\"\"Save raw benchmark data to CSV\"\"\"\n        if filename is None:\n            timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n            filename = os.path.join(self.results_dir, f\"benchmark_data_{timestamp}.csv\")\n\n        if not self.results:\n            print(\"No results to save\")\n            return\n\n        df = pd.DataFrame([asdict(r) for r in self.results])\n        df.to_csv(filename, index=False)\n        print(f\"Raw data saved to {filename}\")\n\n\nasync def main():\n    \"\"\"Main function to run the benchmark\"\"\"\n    print(\"🚀 LLM Benchmark Suite\")\n    print(\"=\"*50)\n\n    # Load API keys\n    print(\"Setting up API keys...\")\n    api_keys = APIKeyManager.load_from_env()\n\n    # Fallback to .env file\n    if not any(api_keys.values()):\n        print(\"No environment variables found, trying .env file...\")\n        api_keys = APIKeyManager.load_from_file('.env')\n\n    # Debug: Print which keys were found (without revealing the actual keys)\n    print(\"API keys found:\")\n    for provider, key in api_keys.items():\n        status = \"✓\" if key else \"✗\"\n        print(f\"  {status} {provider}: {'Available' if key else 'Not found'}\")\n\n    # Additional debugging for environment variables\n    print(\"\\nEnvironment variable check:\")\n    env_vars_to_check = [\n        'DEEPSEEK_API_KEY', 'DEEPSEEK_TOKEN', 'DEEPSEEK_KEY', 'DS_API_KEY',\n        'XAI_API_KEY', 'XAI_TOKEN', 'XAI_KEY', 'GROK_API_KEY',\n        'GOOGLE_API_KEY', 'GEMINI_API_KEY',\n        'MOONSHOT_API_KEY', 'MOONSHOT_TOKEN',\n        'OPENROUTER_API_KEY', 'OPENROUTER_TOKEN',\n        'MISTRAL_API_KEY'\n    ]\n    for var in env_vars_to_check:\n        value = os.getenv(var)\n        print(f\"  {var}: {'Set' if value else 'Not set'}\")\n\n    # Initialise benchmark with results directory\n    benchmark = LLMBenchmark(results_dir=\"results\")\n\n    # Add models based on available API keys\n    models_added = 0\n\n    if api_keys.get('openai'):\n        try:\n            benchmark.add_model(OpenAIInterface(api_keys['openai'], \"gpt-4.1\"))\n            print(\"✓ Added OpenAI GPT-4.1\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add OpenAI: {e}\")\n\n    if api_keys.get('anthropic'):\n        try:\n            benchmark.add_model(AnthropicInterface(api_keys['anthropic'], \"claude-sonnet-4-20250514\"))\n            print(\"✓ Added Anthropic Claude 4 Sonnet\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add Anthropic: {e}\")\n\n    if api_keys.get('deepseek'):\n        try:\n            benchmark.add_model(DeepSeekInterface(api_keys['deepseek'], \"deepseek-chat\"))\n            print(\"✓ Added DeepSeek V3\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add DeepSeek: {e}\")\n\n    if api_keys.get('xai'):\n        try:\n            benchmark.add_model(XAIInterface(api_keys['xai'], \"grok-4-0709\"))\n            print(\"✓ Added xAI Grok\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add xAI: {e}\")\n\n    if api_keys.get('google'):\n        try:\n            benchmark.add_model(GeminiInterface(api_keys['google'], \"gemini-2.5-pro\"))\n            print(\"✓ Added Google Gemini 2.5 Pro\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add Google Gemini: {e}\")\n\n    if api_keys.get('moonshot'):\n        try:\n            benchmark.add_model(MoonshotInterface(api_keys['moonshot'], \"moonshot-v1-8k\"))\n            print(\"✓ Added Moonshot AI\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add Moonshot: {e}\")\n\n    if api_keys.get('openrouter'):\n        try:\n            benchmark.add_model(QwenInterface(api_keys['openrouter'], \"qwen/qwen3-32b:free\"))\n            print(\"✓ Added Qwen3 32B (via OpenRouter)\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add Qwen3 32B: {e}\")\n\n    if api_keys.get('mistral'):\n        try:\n            benchmark.add_model(MistralInterface(api_keys['mistral'], \"mistral-medium-2505\"))\n            print(\"✓ Added Mistral Medium 2505\")\n            models_added += 1\n        except Exception as e:\n            print(f\"✗ Failed to add Mistral: {e}\")\n\n    if models_added == 0:\n        print(\"\\n❌ No models were successfully added!\")\n        print(\"\\nTo fix this, set your API keys using one of these methods:\")\n        print(\"\\n1. Environment variables:\")\n        print(\"   export OPENAI_API_KEY='sk-...'\")\n        print(\"   export ANTHROPIC_API_KEY='sk-ant-...'\")\n        print(\"   export DEEPSEEK_API_KEY='sk-...'\")\n        print(\"   export XAI_API_KEY='xai-...'\")\n        print(\"   export GOOGLE_API_KEY='AIza...'\")\n        print(\"   export MOONSHOT_API_KEY='sk-...'\")\n        print(\"   export OPENROUTER_API_KEY='sk-or-v1-...'\")\n        print(\"   export MISTRAL_API_KEY='...'\")\n        print(\"\\n2. Create a .env file:\")\n        print(\"   OPENAI_API_KEY=sk-...\")\n        print(\"   ANTHROPIC_API_KEY=sk-ant-...\")\n        print(\"   DEEPSEEK_API_KEY=sk-...\")\n        print(\"   XAI_API_KEY=xai-...\")\n        print(\"   GOOGLE_API_KEY=AIza...\")\n        print(\"   MOONSHOT_API_KEY=sk-...\")\n        print(\"   OPENROUTER_API_KEY=sk-or-v1-...\")\n        print(\"   MISTRAL_API_KEY=...\")\n        print(\"\\n3. Set them directly in the code (testing only)\")\n        return\n\n    print(f\"\\n🚀 Successfully configured {models_added} models\")\n\n    # Create standard benchmark tasks\n    benchmark.create_standard_tasks()\n    print(f\"Created {len(benchmark.tasks)} benchmark tasks\")\n\n    # Run benchmark\n    print(\"\\nStarting benchmark run...\")\n    iterations = 2  # Adjust as needed\n    await benchmark.run_benchmark(iterations=iterations)\n\n    # Analyse and generate report\n    print(\"\\nAnalysing results...\")\n    analysis = benchmark.analyse_results()\n\n    # Print summary table\n    benchmark.print_summary_table()\n\n    # Generate detailed report\n    benchmark.generate_report()\n\n    # Save raw data\n    benchmark.save_raw_data()\n\n    # Create visualisations\n    print(\"\\nCreating visualisations...\")\n    benchmark.create_visualisations()\n\n    print(\"\\n✅ Benchmark completed successfully!\")\n    print(f\"📁 All results saved to '{benchmark.results_dir}' directory\")\n    print(\"📊 Check the directory for:\")\n    print(\"   - JSON report with detailed analysis\")\n    print(\"   - CSV file with raw data\")\n    print(\"   - PNG charts with visualisations\")\n\n    # Print key insights\n    if 'overall_stats' in analysis:\n        stats = analysis['overall_stats']\n        print(f\"\\n🏆 Best Overall Model: {stats['best_overall_model']}\")\n        print(f\"📈 Overall Average Quality: {stats['overall_avg_quality_score']:.1f}%\")\n        print(f\"💰 Total Cost: ${stats['total_cost']:.4f}\")\n        print(f\"⚡ Average Latency: {stats['avg_latency']:.2f}s\")\n\n\nif __name__ == \"__main__\":\n    import sys\n\n    if len(sys.argv) > 1 and sys.argv[1] == \"setup\":\n        setup_environment()\n    else:\n        asyncio.run(main())\n"}}, "scripts/utils.py": {"0a1da09e3f670e78794b668f37e222ad1c521216": {"t": 1785674122.2411191, "n": 14704, "h": "8d7da4006f6855c2", "c": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\n\"\"\"\n@name: utils.py\n@author: Finbarrs Oketunji\n@contact: f@finbarrs.eu\n@time: Wednesday July 09 23:05:15 2025\n@updated: Tuesday July 15 18:02:00 2025\n@desc: Utilities for LLM benchmarking including evaluation and API key management\n\"\"\"\n\nimport os\nfrom typing import Dict, List, Optional\nfrom dataclasses import dataclass\nfrom datetime import datetime\n\n\n@dataclass\nclass BenchmarkResult:\n    model_name: str\n    task_name: str\n    response: str\n    latency: float\n    input_tokens: int\n    output_tokens: int\n    cost_estimate: float\n    timestamp: datetime\n    quality_score: float  # Percentage score (0-100)\n    error: Optional[str] = None\n\n\n@dataclass\nclass BenchmarkTask:\n    name: str\n    prompt: str\n    category: str\n    expected_length: str  # short, medium, long\n    evaluation_criteria: List[str]\n\n\nclass QualityEvaluator:\n    \"\"\"Evaluates response quality based on task criteria\"\"\"\n\n    @staticmethod\n    def evaluate_response(task: BenchmarkTask, response: str) -> float:\n        \"\"\"Evaluate response quality and return percentage score (0-100)\"\"\"\n        if not response or response.strip() == \"\":\n            return 0.0\n\n        score = 0.0\n\n        # Base score for having a response\n        score += 20.0\n\n        # Length appropriateness (0-20 points)\n        length_score = QualityEvaluator._evaluate_length(task.expected_length, response)\n        score += length_score\n\n        # Task-specific evaluation (0-60 points)\n        task_score = QualityEvaluator._evaluate_by_task_type(task, response)\n        score += task_score\n\n        return min(score, 100.0)\n\n    @staticmethod\n    def _evaluate_length(expected_length: str, response: str) -> float:\n        \"\"\"Evaluate if response length matches expectations (0-20 points)\"\"\"\n        word_count = len(response.split())\n\n        length_targets = {\n            \"short\": (20, 100),  # 20-100 words\n            \"medium\": (100, 400),  # 100-400 words\n            \"long\": (300, 800),  # 300-800 words\n        }\n\n        if expected_length not in length_targets:\n            return 10.0  # Default score\n\n        min_words, max_words = length_targets[expected_length]\n\n        if min_words <= word_count <= max_words:\n            return 20.0\n        elif word_count < min_words:\n            # Penalise for being too short\n            ratio = word_count / min_words\n            return 20.0 * ratio\n        else:\n            # Moderate penalty for being too long\n            excess_ratio = (word_count - max_words) / max_words\n            penalty = min(excess_ratio * 5, 10)  # Max 10 point penalty\n            return max(20.0 - penalty, 5.0)\n\n    @staticmethod\n    def _evaluate_by_task_type(task: BenchmarkTask, response: str) -> float:\n        \"\"\"Evaluate based on task category and criteria (0-60 points)\"\"\"\n        category = task.category.lower()\n\n        if category == \"programming\":\n            return QualityEvaluator._evaluate_code(response)\n        elif category == \"mathematics\":\n            return QualityEvaluator._evaluate_math(response)\n        elif category == \"creative\":\n            return QualityEvaluator._evaluate_creative(response)\n        elif category == \"analysis\":\n            return QualityEvaluator._evaluate_analysis(response)\n        elif category == \"logic\":\n            return QualityEvaluator._evaluate_logic(response)\n        elif category == \"communication\":\n            return QualityEvaluator._evaluate_communication(response)\n        else:\n            return QualityEvaluator._evaluate_general(response)\n\n    @staticmethod\n    def _evaluate_code(response: str) -> float:\n        \"\"\"Evaluate programming responses (0-60 points)\"\"\"\n        score = 0.0\n\n        # Check for code blocks or function definitions\n        if \"def \" in response or \"function\" in response or \"```\" in response:\n            score += 15.0\n\n        # Check for error handling\n        if any(\n            term in response.lower()\n            for term in [\"try\", \"except\", \"error\", \"validation\"]\n        ):\n            score += 10.0\n\n        # Check for documentation\n        if any(term in response for term in ['\"\"\"', \"'''\", \"//\", \"#\"]):\n            score += 10.0\n\n        # Check for proper structure\n        if \"return\" in response and (\n            \"if\" in response or \"for\" in response or \"while\" in response\n        ):\n            score += 15.0\n\n        # Check for examples or test cases\n        if any(term in response.lower() for term in [\"example\", \"test\", \"usage\"]):\n            score += 10.0\n\n        return score\n\n    @staticmethod\n    def _evaluate_math(response: str) -> float:\n        \"\"\"Evaluate mathematical reasoning (0-60 points)\"\"\"\n        score = 0.0\n\n        # Check for step-by-step approach\n        if any(\n            term in response.lower()\n            for term in [\"step\", \"first\", \"then\", \"next\", \"finally\"]\n        ):\n            score += 20.0\n\n        # Check for calculations\n        if any(char in response for char in \"=+-*/\") and any(\n            char.isdigit() for char in response\n        ):\n            score += 15.0\n\n        # Check for units or measurements\n        if any(\n            term in response.lower()\n            for term in [\"miles\", \"hours\", \"mph\", \"speed\", \"time\"]\n        ):\n            score += 15.0\n\n        # Check for clear conclusion\n        if any(\n            term in response.lower()\n            for term in [\"therefore\", \"total\", \"answer\", \"result\"]\n        ):\n            score += 10.0\n\n        return score\n\n    @staticmethod\n    def _evaluate_creative(response: str) -> float:\n        \"\"\"Evaluate creative writing (0-60 points)\"\"\"\n        score = 0.0\n\n        # Check for narrative elements\n        narrative_indicators = [\n            \"said\",\n            \"felt\",\n            \"looked\",\n            \"thought\",\n            \"realised\",\n            \"discovered\",\n        ]\n        if any(term in response.lower() for term in narrative_indicators):\n            score += 20.0\n\n        # Check for emotional content\n        emotion_words = [\n            \"happy\",\n            \"sad\",\n            \"surprised\",\n            \"confused\",\n            \"excited\",\n            \"worried\",\n            \"wonder\",\n        ]\n        if any(term in response.lower() for term in emotion_words):\n            score += 15.0\n\n        # Check for descriptive language\n        if len([word for word in response.split() if len(word) > 6]) > 10:\n            score += 15.0\n\n        # Check for dialogue or direct speech\n        if '\"' in response or \"'\" in response:\n            score += 10.0\n\n        return score\n\n    @staticmethod\n    def _evaluate_analysis(response: str) -> float:\n        \"\"\"Evaluate analytical responses (0-60 points)\"\"\"\n        score = 0.0\n\n        # Check for key concepts mentioned\n        key_terms = [\"correlation\", \"causation\", \"relationship\", \"data\", \"statistics\"]\n        if any(term in response.lower() for term in key_terms):\n            score += 20.0\n\n        # Check for examples\n        if any(\n            term in response.lower()\n            for term in [\"example\", \"instance\", \"case\", \"such as\"]\n        ):\n            score += 15.0\n\n        # Check for practical applications\n        if any(\n            term in response.lower()\n            for term in [\"practical\", \"real-world\", \"application\", \"important\"]\n        ):\n            score += 15.0\n\n        # Check for clear structure\n        if any(\n            term in response.lower()\n            for term in [\"first\", \"second\", \"however\", \"furthermore\"]\n        ):\n            score += 10.0\n\n        return score\n\n    @staticmethod\n    def _evaluate_logic(response: str) -> float:\n        \"\"\"Evaluate logical reasoning (0-60 points)\"\"\"\n        score = 0.0\n\n        # Check for logical structure\n        if any(\n            term in response.lower()\n            for term in [\"premise\", \"conclusion\", \"therefore\", \"because\"]\n        ):\n            score += 20.0\n\n        # Check for reasoning process\n        if any(\n            term in response.lower()\n            for term in [\"valid\", \"invalid\", \"logical\", \"reasoning\"]\n        ):\n            score += 15.0\n\n        # Check for clear explanation\n        if len(response.split()) > 50:  # Adequate length for explanation\n            score += 15.0\n\n        # Check for structured argument\n        if any(\n            term in response.lower() for term in [\"cannot\", \"can\", \"conclude\", \"assume\"]\n        ):\n            score += 10.0\n\n        return score\n\n    @staticmethod\n    def _evaluate_communication(response: str) -> float:\n        \"\"\"Evaluate communication advice (0-60 points)\"\"\"\n        score = 0.0\n\n        # Check for key communication concepts\n        comm_terms = [\n            \"listen\",\n            \"clear\",\n            \"communicate\",\n            \"understand\",\n            \"feedback\",\n            \"respect\",\n        ]\n        if any(term in response.lower() for term in comm_terms):\n            score += 20.0\n\n        # Check for actionable advice\n        if any(\n            term in response.lower()\n            for term in [\"should\", \"can\", \"try\", \"practice\", \"avoid\"]\n        ):\n            score += 15.0\n\n        # Check for professional context\n        if any(\n            term in response.lower()\n            for term in [\"professional\", \"workplace\", \"colleague\", \"meeting\"]\n        ):\n            score += 15.0\n\n        # Check for organisation\n        if any(term in response for term in [\"1.\", \"2.\", \"•\", \"-\", \"first\", \"second\"]):\n            score += 10.0\n\n        return score\n\n    @staticmethod\n    def _evaluate_general(response: str) -> float:\n        \"\"\"General evaluation for uncategorised tasks (0-60 points)\"\"\"\n        score = 0.0\n\n        # Basic completeness\n        word_count = len(response.split())\n        if word_count > 20:\n            score += 20.0\n        elif word_count > 10:\n            score += 10.0\n\n        # Check for structured thinking\n        if any(\n            term in response.lower()\n            for term in [\"because\", \"therefore\", \"however\", \"although\"]\n        ):\n            score += 15.0\n\n        # Check for examples or specifics\n        if any(\n            term in response.lower()\n            for term in [\"example\", \"such as\", \"specifically\", \"instance\"]\n        ):\n            score += 15.0\n\n        # Basic coherence check (sentences and punctuation)\n        sentence_count = response.count(\".\") + response.count(\"!\") + response.count(\"?\")\n        if sentence_count >= 3:\n            score += 10.0\n\n        return score\n\n\nclass APIKeyManager:\n    \"\"\"Secure API key management\"\"\"\n\n    @staticmethod\n    def load_from_env() -> Dict[str, Optional[str]]:\n        \"\"\"Load API keys from environment variables\"\"\"\n        return {\n            \"openai\": os.getenv(\"OPENAI_API_KEY\"),\n            \"anthropic\": os.getenv(\"ANTHROPIC_API_KEY\"),\n            \"deepseek\": os.getenv(\"DEEPSEEK_API_KEY\") or os.getenv(\"DEEPSEEK_TOKEN\"),\n            \"xai\": os.getenv(\"XAI_API_KEY\") or os.getenv(\"XAI_TOKEN\"),\n            \"google\": os.getenv(\"GOOGLE_API_KEY\") or os.getenv(\"GEMINI_API_KEY\"),\n            \"moonshot\": os.getenv(\"MOONSHOT_API_KEY\") or os.getenv(\"MOONSHOT_TOKEN\"),\n            \"openrouter\": os.getenv(\"OPENROUTER_API_KEY\") or os.getenv(\"OPENROUTER_TOKEN\"),\n            \"mistral\": os.getenv(\"MISTRAL_API_KEY\"),\n        }\n\n    @staticmethod\n    def load_from_file(config_file: str = \".env\") -> Dict[str, Optional[str]]:\n        \"\"\"Load API keys from configuration file\"\"\"\n        keys = {}\n        try:\n            with open(config_file, \"r\") as f:\n                for line in f:\n                    line = line.strip()\n                    if line and \"=\" in line and not line.startswith(\"#\"):\n                        key, value = line.split(\"=\", 1)\n                        key_name = key.strip().lower()\n                        # Normalise key names\n                        if \"openai\" in key_name:\n                            key_name = \"openai\"\n                        elif \"anthropic\" in key_name:\n                            key_name = \"anthropic\"\n                        elif \"deepseek\" in key_name:\n                            key_name = \"deepseek\"\n                        elif \"xai\" in key_name:\n                            key_name = \"xai\"\n                        elif \"google\" in key_name or \"gemini\" in key_name:\n                            key_name = \"google\"\n                        elif \"moonshot\" in key_name:\n                            key_name = \"moonshot\"\n                        elif \"openrouter\" in key_name:\n                            key_name = \"openrouter\"\n                        elif \"mistral\" in key_name:\n                            key_name = \"mistral\"\n\n                        keys[key_name] = value.strip().strip('\"').strip(\"'\")\n        except FileNotFoundError:\n            print(f\"Configuration file {config_file} not found\")\n        return keys\n\n\ndef setup_environment():\n    \"\"\"Helper function to guide users through API key setup\"\"\"\n    print(\"=== API Key Setup Guide ===\\n\")\n\n    print(\"You need API keys from these providers:\")\n    print(\"1. OpenAI: https://platform.openai.com/api-keys\")\n    print(\"2. Anthropic: https://console.anthropic.com/\")\n    print(\"3. DeepSeek: https://platform.deepseek.com/\")\n    print(\"4. xAI: https://x.ai/api\")\n    print(\"5. Google: https://aistudio.google.com/app/apikey\")\n    print(\"6. Moonshot: https://platform.moonshot.ai/\")\n    print(\"7. OpenRouter: https://openrouter.ai/keys\")\n    print(\"8. Mistral: https://console.mistral.ai/\")\n\n    print(\"\\n=== Recommended Setup Method ===\")\n    print(\"Create a .env file in your project directory:\")\n    print()\n    print(\"# .env file content\")\n    print(\"OPENAI_API_KEY=sk-proj-...\")\n    print(\"ANTHROPIC_API_KEY=sk-ant-...\")\n    print(\"DEEPSEEK_API_KEY=sk-...\")\n    print(\"XAI_API_KEY=xai-...\")\n    print(\"GOOGLE_API_KEY=AIza...\")\n    print(\"MOONSHOT_API_KEY=sk-...\")\n    print(\"OPENROUTER_API_KEY=sk-or-v1-...\")\n    print(\"MISTRAL_API_KEY=...\")\n    print()\n    print(\"Then run: python benchmark.py\")\n\n    print(\"\\n=== Alternative: Environment Variables ===\")\n    print(\"Linux/Mac:\")\n    print(\"export OPENAI_API_KEY='sk-proj-...'\")\n    print(\"export ANTHROPIC_API_KEY='sk-ant-...'\")\n    print(\"export DEEPSEEK_API_KEY='sk-...'\")\n    print(\"export XAI_API_KEY='xai-...'\")\n    print(\"export GOOGLE_API_KEY='AIza...'\")\n    print(\"export OPENROUTER_API_KEY='sk-or-v1-...'\")\n    print(\"export MISTRAL_API_KEY='...'\")\n    print()\n    print(\"Windows:\")\n    print(\"set OPENAI_API_KEY=sk-proj-...\")\n    print(\"set ANTHROPIC_API_KEY=sk-ant-...\")\n    print(\"set DEEPSEEK_API_KEY=sk-...\")\n    print(\"set XAI_API_KEY=xai-...\")\n    print(\"set GOOGLE_API_KEY=AIza...\")\n    print(\"set OPENROUTER_API_KEY=sk-or-v1-...\")\n    print(\"set MISTRAL_API_KEY=...\")\n"}}}}