{"repo": "abisf/ragtestercli", "files": {"cli.py": {"cfd225ca82b2ae90311177e396e4953fd10c42eb": {"t": 1785674360.5853372, "n": 35563, "h": "cac937aaebd28acf", "c": "#!/usr/bin/env python3\n\"\"\"\nRAGTesterCLI - Unified RAG Evaluation Tool\n\"\"\"\n\nimport os\nimport typer\nfrom typing import Optional\nfrom runner import run_evaluation\nfrom dotenv import load_dotenv\n\n# Load environment variables from .env file\nload_dotenv()\n\napp = typer.Typer()\n\n# Provider detection mapping\nPROVIDER_MAP = {\n    \"gpt-\": \"OPENAI\",\n    \"claude-\": \"ANTHROPIC\",\n    \"anthropic/\": \"ANTHROPIC\",\n    \"gemini\": \"GOOGLE\",\n    \"google/\": \"GOOGLE\",\n    \"mistral\": \"MISTRAL\",\n    \"llama\": \"TOGETHER\",\n    \"meta-llama\": \"TOGETHER\",\n}\n\ndef clean_model_name(model_name: str, api_base: Optional[str] = None) -> str:\n    \"\"\"Clean model name by removing provider prefixes, except for OpenRouter which needs them.\"\"\"\n    # For OpenRouter, keep provider prefixes as they're part of the model ID\n    if api_base and \"openrouter.ai\" in api_base:\n        return model_name  # Keep as-is for OpenRouter\n    \n    # Remove common provider prefixes for other APIs\n    prefixes_to_remove = [\"google/\", \"anthropic/\", \"openai/\", \"openrouter/\"]\n    \n    for prefix in prefixes_to_remove:\n        if model_name.startswith(prefix):\n            return model_name[len(prefix):]\n    \n    return model_name\n\ndef detect_provider(model_name: str, api_base: Optional[str] = None) -> str:\n    \"\"\"Detect provider from model name and API base, return env var name.\"\"\"\n    # Special case: If using OpenRouter, always use OPENROUTER_API_KEY\n    if api_base and \"openrouter.ai\" in api_base:\n        return \"OPENROUTER_API_KEY\"\n    \n    # Otherwise, detect from model name\n    for prefix, prov in PROVIDER_MAP.items():\n        if model_name.lower().startswith(prefix):\n            return prov + \"_API_KEY\"\n    # Default to OpenAI for unknown models\n    return \"OPENAI_API_KEY\"\n\n@app.command()\ndef test(\n    input: str = typer.Option(..., \"--input\", \"-i\", help=\"Path to input JSON file\"),\n    metric: str = typer.Option(..., \"--metric\", \"-m\", help=\"Evaluation metric to use\"),\n    llm_model: str = typer.Option(..., \"--llm-model\", help=\"Model name, e.g. gpt-4, claude-3-sonnet, gemini-pro\"),\n    api_key: Optional[str] = typer.Option(None, \"--api-key\", help=\"API key for your chosen provider (or use .env)\"),\n    api_base: Optional[str] = typer.Option(None, \"--api-base\", help=\"(Optional) Custom API base URL (or use .env)\"),\n    output: Optional[str] = typer.Option(None, \"--output\", \"-o\", help=\"Output file path (optional)\"),\n    format: str = typer.Option(\"json\", \"--format\", \"-f\", help=\"Output format: json, csv, table\"),\n    verbose: bool = typer.Option(False, \"--verbose\", \"-v\", help=\"Enable verbose output\"),\n):\n    \"\"\"\n    RAGTesterCLI - Unified RAG evaluation with support for multiple providers\n    \n    Examples:\n      # OpenAI\n      ragtester test --llm-model gpt-4 --api-key sk-...\n      \n      # OpenRouter (any model)\n      ragtester test --llm-model google/gemini-pro --api-key sk-or-v1-... --api-base https://openrouter.ai/api/v1\n      \n      # Direct Anthropic\n      ragtester test --llm-model claude-3-sonnet --api-key sk-ant-...\n      \n      # Direct Google\n      ragtester test --llm-model gemini-pro --api-key AIzaSy... --api-base https://generativelanguage.googleapis.com/v1\n    \"\"\"\n    \n    # 1) Get API key from CLI or environment\n    if api_key:\n        # Use CLI-provided API key\n        key_var = detect_provider(llm_model, api_base)\n        os.environ[key_var] = api_key\n        os.environ[\"OPENAI_API_KEY\"] = api_key  # Mirror for RAGAS\n    else:\n        # Use environment variables from .env\n        key_var = detect_provider(llm_model, api_base)\n        env_api_key = os.getenv(key_var)\n        if not env_api_key:\n            typer.echo(f\"Error: No API key found. Set {key_var} in .env file or use --api-key\")\n            raise typer.Exit(1)\n        api_key = env_api_key\n    \n    # 3) Clean model name for actual API calls (remove provider prefixes)\n    clean_model = clean_model_name(llm_model, api_base)\n    os.environ[\"RAGCLI_LLM_MODEL\"] = clean_model\n    \n    # 4) Set custom API base if provided\n    if api_base:\n        os.environ[\"OPENAI_API_BASE\"] = api_base\n    else:\n        # Try to get from environment\n        env_api_base = os.getenv(\"OPENAI_API_BASE\")\n        if env_api_base:\n            api_base = env_api_base\n    \n    # Special handling for OpenRouter - set LiteLLM-specific vars\n    if api_base and \"openrouter.ai\" in api_base and api_key.startswith(\"sk-or-v1-\"):\n        os.environ[\"LITELLM_API_KEY\"] = api_key\n        os.environ[\"LITELLM_API_BASE\"] = api_base\n        os.environ[\"OPENROUTER_API_KEY\"] = api_key  # Some versions use this\n        print(f\"Set LiteLLM vars for OpenRouter\")\n        \n    if verbose:\n        typer.echo(f\"RAGTesterCLI Configuration:\")\n        typer.echo(f\"  Original Model: {llm_model}\")\n        typer.echo(f\"  Clean Model: {clean_model}\")\n        typer.echo(f\"  Provider Key: {key_var}\")\n        typer.echo(f\"  API Base: {api_base or 'default'}\")\n        typer.echo(f\"  Metric: {metric}\")\n        typer.echo()\n\n    # 5) Run evaluation with unified config\n    model_config = {\n        \"llm_model\": llm_model,\n        \"api_key\": api_key,\n        \"api_base\": api_base\n    }\n    \n    try:\n        result = run_evaluation(\n            input_file=input,\n            metric=metric,\n            output_file=output,\n            output_format=format,\n            verbose=verbose,\n            model_config=model_config\n        )\n        \n        if result:\n            typer.echo(result)\n            \n    except Exception as e:\n        error_msg = str(e)\n        \n        # Check for quota/rate limit errors and show friendly message\n        if any(keyword in error_msg.lower() for keyword in [\"quota\", \"rate limit\", \"insufficient_quota\", \"billing\"]):\n            typer.echo(f\"\\nError: API quota exceeded\")\n            typer.echo(f\"Please check your billing and usage limits, or try again later.\")\n            typer.echo(f\"Details: {error_msg}\")\n            raise typer.Exit(1)\n        elif \"authentication\" in error_msg.lower() or \"unauthorized\" in error_msg.lower():\n            typer.echo(f\"\\nError: Authentication failed\")\n            typer.echo(f\"Please check your API key and try again.\")\n            typer.echo(f\"Details: {error_msg}\")\n            raise typer.Exit(1)\n        else:\n            typer.echo(f\"\\nError: {error_msg}\")\n            raise typer.Exit(1)\n\n@app.command()\ndef analyze(\n    input: str = typer.Option(..., \"--input\", \"-i\", help=\"Path to input JSON file\"),\n    metric: str = typer.Option(\"faithfulness\", \"--metric\", \"-m\", help=\"Analysis type: faithfulness, context_precision, or hallucination\"),\n    llm_model: str = typer.Option(\"gpt-3.5-turbo\", \"--llm-model\", help=\"Model name for analysis\"),\n    api_key: Optional[str] = typer.Option(None, \"--api-key\", help=\"API key for your chosen provider (or use .env)\"),\n    api_base: Optional[str] = typer.Option(None, \"--api-base\", help=\"(Optional) Custom API base URL (or use .env)\"),\n    verbose: bool = typer.Option(False, \"--verbose\", \"-v\", help=\"Enable verbose output\"),\n):\n    \"\"\"\n    Analyze RAG metrics with detailed breakdowns.\n    \n    Faithfulness: Shows claim-by-claim analysis\n    Context Precision: Shows chunk-by-chunk relevance analysis with Precision@k\n    Hallucination: Shows detailed hallucination detection analysis\n    \"\"\"\n    \n    # Setup API keys (same logic as test command)\n    if api_key:\n        key_var = detect_provider(llm_model, api_base)\n        os.environ[key_var] = api_key\n        os.environ[\"OPENAI_API_KEY\"] = api_key\n    else:\n        key_var = detect_provider(llm_model, api_base)\n        env_api_key = os.getenv(key_var)\n        if not env_api_key:\n            typer.echo(f\"Error: No API key found. Set {key_var} in .env file or use --api-key\")\n            raise typer.Exit(1)\n        api_key = env_api_key\n    \n    clean_model = clean_model_name(llm_model, api_base)\n    os.environ[\"RAGCLI_LLM_MODEL\"] = clean_model\n    \n    if api_base:\n        os.environ[\"OPENAI_API_BASE\"] = api_base\n    \n    if api_base and \"openrouter.ai\" in api_base and api_key.startswith(\"sk-or-v1-\"):\n        os.environ[\"LITELLM_API_KEY\"] = api_key\n        os.environ[\"LITELLM_API_BASE\"] = api_base\n        os.environ[\"OPENROUTER_API_KEY\"] = api_key\n        print(f\"Set LiteLLM vars for OpenRouter\")\n    \n    # Load input data\n    from runner import load_input_data\n    data_list = load_input_data(input)\n    \n    # Build model config for evaluator\n    model_config = {\n        \"llm_model\": llm_model,\n        \"api_key\": api_key,\n        \"api_base\": api_base\n    }\n    \n    # Initialize appropriate evaluator based on metric\n    if metric == \"faithfulness\":\n        from evaluators.ragas_faithfulness import RagasFaithfulnessEvaluator\n        evaluator = RagasFaithfulnessEvaluator(model_config)\n        typer.echo(\"RAGAS Faithfulness - Detailed Claim Analysis\")\n        typer.echo(\"=\" * 50)\n        \n        for i, data in enumerate(data_list, 1):\n            typer.echo(f\"\\nTest Case {i}:\")\n            typer.echo(\"-\" * 30)\n            \n            analysis = evaluator.get_detailed_analysis(data)\n            \n            typer.echo(f\"Question: {data.get('question', '')}\")\n            typer.echo(f\"Context: {' '.join(data.get('context', []))}\")\n            typer.echo(f\"Answer: {data.get('answer', '')}\")\n            typer.echo()\n            \n            if \"error\" in analysis:\n                typer.echo(f\"Error: {analysis['error']}\")\n                continue\n            \n            # Display formula explanation\n            if 'formula_explanation' in analysis:\n                formula = analysis['formula_explanation']\n                typer.echo(\"📊 Faithfulness Formula:\")\n                typer.echo(f\"  {formula['faithfulness_formula']}\")\n                typer.echo(f\"  Calculation: {formula['calculation']}\")\n                typer.echo(f\"  Interpretation: {formula['interpretation']}\")\n                typer.echo()\n            \n            # Display step-by-step verification process\n            if 'verification_steps' in analysis:\n                typer.echo(\"🔍 Claim Analysis:\")\n                for step in analysis['verification_steps']:\n                    status_icon = \"✓\" if step['is_supported'] else \"✗\"\n                    status_text = \"SUPPORTED\" if step['is_supported'] else \"NOT SUPPORTED\"\n                    typer.echo(f\"  {status_icon} Claim {step['step_number']}: {step['claim']}\")\n                    typer.echo(f\"    Status: {status_text}\")\n                    typer.echo(f\"    Verification: {step['verification_response']}\")\n                    \n                    # Add brief context explanation\n                    if step['is_supported']:\n                        typer.echo(f\"    Context: Found supporting evidence in retrieved chunks\")\n                    else:\n                        typer.echo(f\"    Context: No supporting evidence found in retrieved chunks\")\n                    typer.echo()\n            \n            # Display enhanced context usage analysis\n            if 'context_usage_analysis' in analysis and 'chunks' in analysis['context_usage_analysis']:\n                chunks = analysis['context_usage_analysis']['chunks']\n                typer.echo(\"Context Usage Analysis:\")\n                for chunk in chunks:\n                    usage_icon = \"✓\" if chunk['usage_status'] != 'unused_or_irrelevant' else \"✗\"\n                    typer.echo(f\"  {usage_icon} Chunk {chunk['chunk_number']}: {chunk['usage_status'].replace('_', ' ').title()}\")\n                    typer.echo(f\"    Text: {chunk['chunk_text'][:80]}{'...' if len(chunk['chunk_text']) > 80 else ''}\")\n                    if chunk['supporting_claims']:\n                        typer.echo(f\"    Supporting: {len(chunk['supporting_claims'])} claims\")\n                typer.echo()\n            \n            # Display diagnostic insights\n            if 'diagnostic_insights' in analysis:\n                insights = analysis['diagnostic_insights']\n                typer.echo(f\"Diagnostic Insights (Severity: {insights.get('severity', 'unknown').upper()}):\")\n                if insights.get('primary_issues'):\n                    typer.echo(\"  Issues Found:\")\n                    for issue in insights['primary_issues']:\n                        typer.echo(f\"    • {issue}\")\n                if insights.get('recommendations'):\n                    typer.echo(\"  Recommendations:\")\n                    for rec in insights['recommendations']:\n                        typer.echo(f\"    • {rec}\")\n                if insights.get('specific_problems'):\n                    typer.echo(\"  Specific Unsupported Claims:\")\n                    for problem in insights['specific_problems']:\n                        typer.echo(f\"    • Claim {problem['claim_number']}: {problem['claim_text']}\")\n                typer.echo()\n            \n            # Display summary verdict prominently for casual users\n            if 'summary_verdict' in analysis:\n                typer.echo(\"📋 Summary Verdict:\")\n                typer.echo(f\"  {analysis['summary_verdict']}\")\n                typer.echo()\n            \n            # Display action recommendations\n            if 'action_recommendations' in analysis:\n                action_rec = analysis['action_recommendations']\n                typer.echo(\"🎯 Action Recommendations:\")\n                typer.echo(f\"  Action: {action_rec.get('action', 'none').replace('_', ' ').title()}\")\n                typer.echo(f\"  Reason: {action_rec.get('reason', 'unknown').replace('_', ' ').title()}\")\n                typer.echo(f\"  Confidence: {action_rec.get('confidence', 'unknown').title()}\")\n                if action_rec.get('specific_actions'):\n                    typer.echo(\"  Specific Actions:\")\n                    for action in action_rec['specific_actions']:\n                        typer.echo(f\"    • {action}\")\n                typer.echo()\n            \n            typer.echo(f\"Summary:\")\n            typer.echo(f\"  Total Claims: {analysis['total_claims']}\")\n            typer.echo(f\"  Supported Claims: {analysis['supported_claims']}\")\n            typer.echo(f\"  Unsupported Claims: {analysis['unsupported_claims']}\")\n            typer.echo(f\"  Faithfulness Score: {analysis['faithfulness_score']:.3f}\")\n            \n            # Display additional metrics\n            if 'summary_statistics' in analysis:\n                stats = analysis['summary_statistics']\n                typer.echo()\n                typer.echo(f\"Additional Metrics:\")\n                typer.echo(f\"  Support Rate: {stats.get('support_rate', 0.0):.1%}\")\n                typer.echo(f\"  Unsupported Rate: {stats.get('unsupported_rate', 0.0):.1%}\")\n                typer.echo(f\"  Claim Density: {stats.get('claim_density', 0.0):.3f} claims/word\")\n                typer.echo(f\"  Average Claim Length: {stats.get('average_claim_length', 0.0):.1f} words\")\n            \n            # Display context usage summary\n            if 'context_usage_analysis' in analysis and 'summary' in analysis['context_usage_analysis']:\n                ctx_summary = analysis['context_usage_analysis']['summary']\n                typer.echo()\n                typer.echo(f\"Context Utilization:\")\n                typer.echo(f\"  Used Chunks: {ctx_summary.get('used_chunks', 0)}/{ctx_summary.get('total_chunks', 0)}\")\n                typer.echo(f\"  Utilization Rate: {ctx_summary.get('context_utilization_rate', 0.0):.1%}\")\n                typer.echo(f\"  Total Claims Supported: {ctx_summary.get('total_claims_supported', 0)}\")\n                typer.echo(f\"  Average Relevance Score: {ctx_summary.get('average_relevance_score', 0.0):.3f}\")\n            \n            typer.echo()\n    \n    elif metric == \"context_precision\":\n        from evaluators.ragas_context_precision import RagasContextPrecisionEvaluator\n        evaluator = RagasContextPrecisionEvaluator(model_config)\n        typer.echo(\"RAGAS Context Precision - Detailed Chunk Analysis\")\n        typer.echo(\"=\" * 50)\n        \n        for i, data in enumerate(data_list, 1):\n            typer.echo(f\"\\nTest Case {i}:\")\n            typer.echo(\"-\" * 30)\n            \n            analysis = evaluator.get_detailed_analysis(data)\n            \n            typer.echo(f\"Question: {data.get('question', '')}\")\n            typer.echo(f\"Answer: {data.get('answer', '')}\")\n            typer.echo(f\"Retrieved Contexts ({analysis.get('total_chunks', 0)} chunks):\")\n            for j, context in enumerate(data.get('context', []), 1):\n                typer.echo(f\"  [{j}] {context}\")\n            typer.echo()\n            \n            if \"error\" in analysis:\n                typer.echo(f\"Error: {analysis['error']}\")\n                continue\n            \n            typer.echo(\"Chunk Relevance Analysis:\")\n            for chunk_data in analysis['chunk_analysis']:\n                status = \"✓ RELEVANT\" if chunk_data['relevant'] else \"✗ NOT RELEVANT\"\n                typer.echo(f\"  Chunk {chunk_data['chunk_number']}: {status}\")\n                typer.echo(f\"    Text: {chunk_data['chunk_text'][:80]}{'...' if len(chunk_data['chunk_text']) > 80 else ''}\")\n            \n            typer.echo()\n            typer.echo(\"Precision@k Calculation:\")\n            for precision_data in analysis['precision_at_k']:\n                typer.echo(f\"  Precision@{precision_data['position']}: {precision_data['true_positives']}/{precision_data['total_chunks']} = {precision_data['precision']}\")\n            \n            typer.echo()\n            \n            # Display retrieval quality analysis\n            if 'retrieval_quality_analysis' in analysis:\n                quality = analysis['retrieval_quality_analysis']\n                typer.echo(\"Retrieval Quality Analysis:\")\n                typer.echo(f\"  Precision Trend: {quality.get('precision_trend', 'unknown').replace('_', ' ').title()}\")\n                typer.echo(f\"  Ranking Quality: {quality.get('ranking_quality', 'unknown').replace('_', ' ').title()}\")\n                typer.echo(f\"  Noise Pattern: {quality.get('noise_pattern', 'unknown').replace('_', ' ').title()}\")\n                typer.echo()\n            \n            # Display simplified diagnostic insights\n            if 'diagnostic_insights' in analysis:\n                insights = analysis['diagnostic_insights']\n                severity = insights.get('severity', 'unknown').upper()\n                if severity in ['HIGH', 'CRITICAL']:\n                    typer.echo(f\"⚠️  {severity} RISK: {insights.get('primary_issues', ['Hallucination detected'])[0]}\")\n                elif severity == 'LOW':\n                    typer.echo(\"✅ Low risk - minor issues detected\")\n            \n            # Display summary verdict prominently for casual users\n            if 'summary_verdict' in analysis:\n                typer.echo(\"📋 Summary Verdict:\")\n                typer.echo(f\"  {analysis['summary_verdict']}\")\n                typer.echo()\n            \n            typer.echo(f\"Summary:\")\n            typer.echo(f\"  Total Chunks: {analysis['total_chunks']}\")\n            typer.echo(f\"  Relevant Chunks: {analysis['relevant_chunks']}\")\n            typer.echo()\n            \n            # Display both precision metrics with clear explanations\n            ragas_precision = analysis.get('ragas_context_precision', analysis.get('context_precision', 0.0))\n            simple_precision = analysis.get('simple_context_precision', 0.0)\n            \n            typer.echo(f\"Context Precision Metrics:\")\n            typer.echo(f\"  📊 RAGAS Context Precision: {ragas_precision}\")\n            typer.echo(f\"      → Measures ranking quality: 'Did we rank relevant chunks early?'\")\n            typer.echo(f\"      → Formula: Σ(Precision@k × v_k) / Σ(v_k)\")\n            \n            # Show detailed calculation if precision_at_k is available\n            if 'precision_at_k' in analysis:\n                typer.echo()\n                typer.echo(\"Detailed RAGAS Calculation:\")\n                precision_data = analysis['precision_at_k']\n                relevant_chunks = analysis.get('relevant_chunks', 0)\n                \n                # Show Precision@k calculations\n                typer.echo(\"  Precision@k Calculations:\")\n                for p_data in precision_data:\n                    k = p_data['position']\n                    precision = p_data['precision']\n                    true_pos = p_data['true_positives']\n                    total = p_data['total_chunks']\n                    typer.echo(f\"    Precision@{k}: {true_pos}/{total} = {precision}\")\n                \n                # Show weighted calculation\n                typer.echo()\n                typer.echo(\"  Weighted Calculation (v_k = 1 for relevant chunks):\")\n                weighted_sum = 0\n                for i, chunk in enumerate(analysis.get('chunk_analysis', [])):\n                    if chunk['relevant']:\n                        k = i + 1\n                        precision_k = precision_data[i]['precision']\n                        weighted_sum += precision_k\n                        typer.echo(f\"    Position {k} (relevant): Precision@{k} × 1 = {precision_k}\")\n                \n                if relevant_chunks > 0:\n                    final_score = weighted_sum / relevant_chunks\n                    typer.echo(f\"    Final Score: {weighted_sum} / {relevant_chunks} = {final_score:.3f}\")\n                else:\n                    typer.echo(f\"    Final Score: 0 (no relevant chunks)\")\n            \n            typer.echo()\n            typer.echo(f\"  🔍 Simple Context Precision: {simple_precision}\")\n            typer.echo(f\"      → Measures retrieval cleanliness: 'How much retrieved content is useful?'\")\n            typer.echo(f\"      → Formula: Relevant Chunks / Total Chunks\")\n            \n            # Display summary statistics\n            if 'summary_statistics' in analysis:\n                stats = analysis['summary_statistics']\n                typer.echo()\n                typer.echo(f\"Additional Metrics:\")\n                typer.echo(f\"  Noise Rate: {stats.get('noise_rate', 0.0):.1%}\")\n                typer.echo(f\"      → Percentage of irrelevant chunks in retrieval\")\n                typer.echo(f\"  Early Precision: {stats.get('early_precision', 0.0):.3f}\")\n                typer.echo(f\"      → Precision at first 2 positions (ranking quality)\")\n                typer.echo(f\"  Late Precision: {stats.get('late_precision', 0.0):.3f}\")\n                typer.echo(f\"      → Precision at last 2 positions (overall noise)\")\n            \n            # Interpretive guidance based on both metrics\n            typer.echo()\n            typer.echo(\"Interpretation:\")\n            if ragas_precision >= 0.8 and simple_precision >= 0.8:\n                typer.echo(\"  ✅ Excellent: Good ranking AND low noise\")\n                typer.echo(\"      → Relevant chunks appear early with minimal irrelevant content\")\n            elif ragas_precision >= 0.8 and simple_precision < 0.5:\n                typer.echo(\"  ⚠️  Good early ranking but noisy retrieval\")\n                typer.echo(\"      → Relevant evidence appears first, but overall retrieval contains significant noise\")\n                typer.echo(\"      → Recommendation: Keep top results, prune or rerank the rest to suppress noise\")\n            elif ragas_precision < 0.5 and simple_precision >= 0.8:\n                typer.echo(\"  ⚠️  Clean retrieval but poor ranking\")\n                typer.echo(\"      → Low noise but relevant chunks appear late in ranking\")\n                typer.echo(\"      → Recommendation: Improve ranking algorithm - top results should be most relevant\")\n            else:\n                typer.echo(\"  ❌ Both ranking and noise need improvement\")\n                typer.echo(\"      → Poor early precision AND high noise content\")\n                typer.echo(\"      → Recommendation: Overhaul both similarity matching and ranking algorithms\")\n            \n            # Display action recommendations if available\n            if 'action_recommendations' in analysis:\n                action_rec = analysis['action_recommendations']\n                typer.echo()\n                typer.echo(\"🎯 Action Recommendations:\")\n                \n                action = action_rec.get('action', 'none')\n                if action == 'maintain':\n                    typer.echo(f\"  Action: ✅ Keep current retrieval system\")\n                    typer.echo(f\"      → Your retrieval is working well\")\n                elif action == 'overhaul_retrieval':\n                    typer.echo(f\"  Action: 🔧 Fix both ranking and noise issues\")\n                    typer.echo(f\"      → Improve similarity matching and ranking algorithms\")\n                elif action == 'improve_ranking':\n                    typer.echo(f\"  Action: 📈 Improve ranking algorithm\")\n                    typer.echo(f\"      → Focus on putting most relevant chunks first\")\n                elif action == 'reduce_noise':\n                    typer.echo(f\"  Action: 🧹 Clean up retrieval noise\")\n                    typer.echo(f\"      → Filter out irrelevant content\")\n                else:\n                    typer.echo(f\"  Action: {action}\")\n                \n                if action_rec.get('target_ranks'):\n                    ranks = action_rec['target_ranks']\n                    typer.echo(f\"  Focus Positions: {ranks}\")\n                    typer.echo(f\"      → These positions need improvement\")\n                \n                if action_rec.get('keep_ranks'):\n                    ranks = action_rec['keep_ranks']\n                    typer.echo(f\"  Keep Positions: {ranks}\")\n                    typer.echo(f\"      → These positions are working well\")\n                \n                reason = action_rec.get('reason', 'unknown')\n                if reason == 'good_performance':\n                    typer.echo(f\"  Reason: ✅ Good overall performance\")\n                elif reason == 'poor_ranking_and_high_noise':\n                    typer.echo(f\"  Reason: ❌ Poor ranking AND high noise\")\n                elif reason == 'good_ranking_high_noise':\n                    typer.echo(f\"  Reason: ⚠️ Good ranking but high noise\")\n                elif reason == 'poor_ranking_low_noise':\n                    typer.echo(f\"  Reason: ⚠️ Poor ranking but low noise\")\n                else:\n                    typer.echo(f\"  Reason: {reason}\")\n                \n                confidence = action_rec.get('confidence', 'unknown')\n                if confidence == 'high':\n                    typer.echo(f\"  Confidence: 🔴 High confidence in recommendation\")\n                elif confidence == 'medium':\n                    typer.echo(f\"  Confidence: 🟡 Medium confidence in recommendation\")\n                elif confidence == 'low':\n                    typer.echo(f\"  Confidence: 🟢 Low confidence in recommendation\")\n                else:\n                    typer.echo(f\"  Confidence: {confidence}\")\n            \n            typer.echo()\n    \n    elif metric == \"hallucination\":\n        from evaluators.ragchecker_hallucination import RAGCheckerHallucinationEvaluator\n        evaluator = RAGCheckerHallucinationEvaluator(model_config)\n        typer.echo(\"RAGChecker Hallucination - Detailed Analysis\")\n        typer.echo(\"=\" * 50)\n        \n        for i, data in enumerate(data_list, 1):\n            typer.echo(f\"\\nTest Case {i}:\")\n            typer.echo(\"-\" * 30)\n            \n            analysis = evaluator.get_detailed_analysis(data)\n            \n            typer.echo(f\"Question: {data.get('question', '')}\")\n            typer.echo(f\"Answer: {data.get('answer', '')}\")\n            typer.echo(f\"Context: {' '.join(data.get('context', []))}\")\n            if data.get('ground_truth'):\n                typer.echo(f\"Ground Truth: {data.get('ground_truth')}\")\n            typer.echo()\n            \n            if \"error\" in analysis:\n                typer.echo(f\"Error: {analysis['error']}\")\n                continue\n            \n            # Display simplified formula explanation\n            if 'formula_explanation' in analysis:\n                formula = analysis['formula_explanation']\n                typer.echo(\"Hallucination Score:\")\n                typer.echo(f\"  {formula['calculation']} = {formula['interpretation']}\")\n                typer.echo()\n            \n            # Display enhanced claim-level analysis\n            if 'claim_analysis' in analysis and 'claims' in analysis['claim_analysis']:\n                claims = analysis['claim_analysis']['claims']\n                typer.echo(\"Claim-Level Analysis:\")\n                for claim in claims:\n                    status_icon = \"✓\" if claim['classification'] == 'correct_grounded' else \"✗\"\n                    typer.echo(f\"  {status_icon} Claim {claim['claim_number']}: {claim['claim_text']}\")\n                    typer.echo(f\"    Classification: {claim['classification'].replace('_', ' ').title()}\")\n                    if claim['context_support']['supported']:\n                        typer.echo(f\"    Context Support: ✓ {claim['context_support']['explanation']}\")\n                    else:\n                        typer.echo(f\"    Context Support: ✗ {claim['context_support']['explanation']}\")\n                    if analysis.get('evaluation_mode') == 'full' and 'ground_truth_support' in claim:\n                        if claim['ground_truth_support']['supported']:\n                            typer.echo(f\"    Ground Truth: ✓ {claim['ground_truth_support']['explanation']}\")\n                        else:\n                            typer.echo(f\"    Ground Truth: ✗ {claim['ground_truth_support']['explanation']}\")\n                    typer.echo()\n            \n            # Display context analysis\n            if 'context_analysis' in analysis and 'chunks' in analysis['context_analysis']:\n                chunks = analysis['context_analysis']['chunks']\n                typer.echo(\"Context Usage Analysis:\")\n                for chunk in chunks:\n                    usage_icon = \"✓\" if chunk['usage_analysis'] != 'irrelevant_noise' else \"✗\"\n                    typer.echo(f\"  {usage_icon} Chunk {chunk['chunk_number']}: {chunk['usage_analysis'].replace('_', ' ').title()}\")\n                    typer.echo(f\"    Text: {chunk['chunk_text'][:80]}{'...' if len(chunk['chunk_text']) > 80 else ''}\")\n                    if chunk['supports_claims']:\n                        claim_nums = [str(c['claim_number']) for c in chunk['supports_claims']]\n                        typer.echo(f\"    Supports Claims: {', '.join(claim_nums)}\")\n                typer.echo()\n            \n            # Display diagnostic insights\n            if 'diagnostic_insights' in analysis:\n                insights = analysis['diagnostic_insights']\n                typer.echo(f\"Diagnostic Insights (Severity: {insights.get('severity', 'unknown').upper()}):\")\n                if insights.get('primary_issues'):\n                    typer.echo(\"  Issues Found:\")\n                    for issue in insights['primary_issues']:\n                        typer.echo(f\"    • {issue}\")\n                if insights.get('recommendations'):\n                    typer.echo(\"  Recommendations:\")\n                    for rec in insights['recommendations']:\n                        typer.echo(f\"    • {rec}\")\n                if insights.get('specific_problems'):\n                    typer.echo(\"  Irrelevant Chunks:\")\n                    for problem in insights['specific_problems']:\n                        typer.echo(f\"    • Chunk {problem['chunk_number']}: {problem['chunk_text']}\")\n                typer.echo()\n            \n            # Display summary verdict prominently for casual users\n            if 'summary_verdict' in analysis:\n                typer.echo(\"Summary Verdict:\")\n                typer.echo(f\"  {analysis['summary_verdict']}\")\n                typer.echo()\n            \n            # Display action recommendations\n            if 'action_recommendations' in analysis:\n                action_rec = analysis['action_recommendations']\n                typer.echo(\"Action Recommendations:\")\n                typer.echo(f\"  Action: {action_rec.get('action', 'none').replace('_', ' ').title()}\")\n                typer.echo(f\"  Reason: {action_rec.get('reason', 'unknown').replace('_', ' ').title()}\")\n                typer.echo(f\"  Confidence: {action_rec.get('confidence', 'unknown').title()}\")\n                if action_rec.get('specific_actions'):\n                    typer.echo(\"  Specific Actions:\")\n                    for action in action_rec['specific_actions']:\n                        typer.echo(f\"    • {action}\")\n                typer.echo()\n            \n            typer.echo(f\"Summary:\")\n            typer.echo(f\"  Total Claims: {analysis.get('summary_statistics', {}).get('total_claims', 0)}\")\n            typer.echo(f\"  Hallucinated Claims: {analysis.get('summary_statistics', {}).get('hallucinated_claims', 0)}\")\n            typer.echo(f\"  Correct Grounded Claims: {analysis.get('summary_statistics', {}).get('correct_grounded_claims', 0)}\")\n            typer.echo(f\"  Context Supported but Incorrect: {analysis.get('summary_statistics', {}).get('context_supported_but_incorrect', 0)}\")\n            typer.echo(f\"  Missing Context Evidence: {analysis.get('summary_statistics', {}).get('missing_context_evidence', 0)}\")\n            typer.echo(f\"  Hallucination Score: {analysis['hallucination_score']:.1f}%\")\n            \n            # Display additional metrics\n            if 'summary_statistics' in analysis:\n                stats = analysis['summary_statistics']\n                typer.echo()\n                typer.echo(f\"Additional Metrics:\")\n                typer.echo(f\"  Hallucination Rate: {stats.get('hallucination_rate', 0.0):.1%}\")\n                typer.echo(f\"  Grounding Rate: {stats.get('grounding_rate', 0.0):.1%}\")\n                typer.echo(f\"  Context Support Rate: {stats.get('context_support_rate', 0.0):.1%}\")\n                typer.echo(f\"  Claim Density: {stats.get('claim_density', 0.0):.3f} claims/word\")\n                typer.echo(f\"  Average Claim Length: {stats.get('average_claim_length', 0.0):.1f} words\")\n            \n            # Display context analysis summary\n            if 'context_analysis' in analysis and 'summary' in analysis['context_analysis']:\n                ctx_summary = analysis['context_analysis']['summary']\n                typer.echo()\n                typer.echo(f\"Context Analysis:\")\n                typer.echo(f\"  Total Chunks: {ctx_summary.get('total_chunks', 0)}\")\n                typer.echo(f\"  Relevant Chunks: {ctx_summary.get('relevant_chunks', 0)}\")\n                typer.echo(f\"  Context Precision: {ctx_summary.get('context_precision', 0.0):.3f}\")\n                typer.echo(f\"  Noise Chunks: {ctx_summary.get('noise_chunks', 0)}\")\n                typer.echo(f\"  Average Relevance Score: {ctx_summary.get('average_relevance_score', 0.0):.3f}\")\n            \n            typer.echo()\n            typer.echo()\n    \n    else:\n        typer.echo(f\"Error: Unknown metric '{metric}'. Available: faithfulness, context_precision, hallucination\")\n        raise typer.Exit(1)\n\n@app.command()\ndef list_metrics():\n    \"\"\"List available evaluation metrics.\"\"\"\n    from evaluators import get_available_metrics\n    \n    typer.echo(\"Available Metrics:\")\n    metrics = get_available_metrics()\n    for name, description in metrics.items():\n        typer.echo(f\"  • {name}: {description}\")\n\n@app.command()  \ndef version():\n    \"\"\"Show version information.\"\"\"\n    typer.echo(\"RAGTesterCLI - Unified RAG Evaluation\")\n    typer.echo(\"  Multi-provider LLM support (OpenAI, Anthropic, Google, OpenRouter)\")\n    typer.echo(\"  True RAGAS & RAGChecker integration\")\n    typer.echo(\"  One API key for any provider\")\n\nif __name__ == \"__main__\":\n    app() "}}}}