1 line
36 KiB
JSON
1 line
36 KiB
JSON
{"repo": "abisf/ragtestercli", "files": {"cli.py": {"cfd225ca82b2ae90311177e396e4953fd10c42eb": {"t": 1785674360.5853372, "n": 35563, "h": "cac937aaebd28acf", "c": "#!/usr/bin/env python3\n\"\"\"\nRAGTesterCLI - Unified RAG Evaluation Tool\n\"\"\"\n\nimport os\nimport typer\nfrom typing import Optional\nfrom runner import run_evaluation\nfrom dotenv import load_dotenv\n\n# Load environment variables from .env file\nload_dotenv()\n\napp = typer.Typer()\n\n# Provider detection mapping\nPROVIDER_MAP = {\n \"gpt-\": \"OPENAI\",\n \"claude-\": \"ANTHROPIC\",\n \"anthropic/\": \"ANTHROPIC\",\n \"gemini\": \"GOOGLE\",\n \"google/\": \"GOOGLE\",\n \"mistral\": \"MISTRAL\",\n \"llama\": \"TOGETHER\",\n \"meta-llama\": \"TOGETHER\",\n}\n\ndef clean_model_name(model_name: str, api_base: Optional[str] = None) -> str:\n \"\"\"Clean model name by removing provider prefixes, except for OpenRouter which needs them.\"\"\"\n # For OpenRouter, keep provider prefixes as they're part of the model ID\n if api_base and \"openrouter.ai\" in api_base:\n return model_name # Keep as-is for OpenRouter\n \n # Remove common provider prefixes for other APIs\n prefixes_to_remove = [\"google/\", \"anthropic/\", \"openai/\", \"openrouter/\"]\n \n for prefix in prefixes_to_remove:\n if model_name.startswith(prefix):\n return model_name[len(prefix):]\n \n return model_name\n\ndef detect_provider(model_name: str, api_base: Optional[str] = None) -> str:\n \"\"\"Detect provider from model name and API base, return env var name.\"\"\"\n # Special case: If using OpenRouter, always use OPENROUTER_API_KEY\n if api_base and \"openrouter.ai\" in api_base:\n return \"OPENROUTER_API_KEY\"\n \n # Otherwise, detect from model name\n for prefix, prov in PROVIDER_MAP.items():\n if model_name.lower().startswith(prefix):\n return prov + \"_API_KEY\"\n # Default to OpenAI for unknown models\n return \"OPENAI_API_KEY\"\n\n@app.command()\ndef test(\n input: str = typer.Option(..., \"--input\", \"-i\", help=\"Path to input JSON file\"),\n metric: str = typer.Option(..., \"--metric\", \"-m\", help=\"Evaluation metric to use\"),\n llm_model: str = typer.Option(..., \"--llm-model\", help=\"Model name, e.g. gpt-4, claude-3-sonnet, gemini-pro\"),\n api_key: Optional[str] = typer.Option(None, \"--api-key\", help=\"API key for your chosen provider (or use .env)\"),\n api_base: Optional[str] = typer.Option(None, \"--api-base\", help=\"(Optional) Custom API base URL (or use .env)\"),\n output: Optional[str] = typer.Option(None, \"--output\", \"-o\", help=\"Output file path (optional)\"),\n format: str = typer.Option(\"json\", \"--format\", \"-f\", help=\"Output format: json, csv, table\"),\n verbose: bool = typer.Option(False, \"--verbose\", \"-v\", help=\"Enable verbose output\"),\n):\n \"\"\"\n RAGTesterCLI - Unified RAG evaluation with support for multiple providers\n \n Examples:\n # OpenAI\n ragtester test --llm-model gpt-4 --api-key sk-...\n \n # OpenRouter (any model)\n ragtester test --llm-model google/gemini-pro --api-key sk-or-v1-... --api-base https://openrouter.ai/api/v1\n \n # Direct Anthropic\n ragtester test --llm-model claude-3-sonnet --api-key sk-ant-...\n \n # Direct Google\n ragtester test --llm-model gemini-pro --api-key AIzaSy... --api-base https://generativelanguage.googleapis.com/v1\n \"\"\"\n \n # 1) Get API key from CLI or environment\n if api_key:\n # Use CLI-provided API key\n key_var = detect_provider(llm_model, api_base)\n os.environ[key_var] = api_key\n os.environ[\"OPENAI_API_KEY\"] = api_key # Mirror for RAGAS\n else:\n # Use environment variables from .env\n key_var = detect_provider(llm_model, api_base)\n env_api_key = os.getenv(key_var)\n if not env_api_key:\n typer.echo(f\"Error: No API key found. Set {key_var} in .env file or use --api-key\")\n raise typer.Exit(1)\n api_key = env_api_key\n \n # 3) Clean model name for actual API calls (remove provider prefixes)\n clean_model = clean_model_name(llm_model, api_base)\n os.environ[\"RAGCLI_LLM_MODEL\"] = clean_model\n \n # 4) Set custom API base if provided\n if api_base:\n os.environ[\"OPENAI_API_BASE\"] = api_base\n else:\n # Try to get from environment\n env_api_base = os.getenv(\"OPENAI_API_BASE\")\n if env_api_base:\n api_base = env_api_base\n \n # Special handling for OpenRouter - set LiteLLM-specific vars\n if api_base and \"openrouter.ai\" in api_base and api_key.startswith(\"sk-or-v1-\"):\n os.environ[\"LITELLM_API_KEY\"] = api_key\n os.environ[\"LITELLM_API_BASE\"] = api_base\n os.environ[\"OPENROUTER_API_KEY\"] = api_key # Some versions use this\n print(f\"Set LiteLLM vars for OpenRouter\")\n \n if verbose:\n typer.echo(f\"RAGTesterCLI Configuration:\")\n typer.echo(f\" Original Model: {llm_model}\")\n typer.echo(f\" Clean Model: {clean_model}\")\n typer.echo(f\" Provider Key: {key_var}\")\n typer.echo(f\" API Base: {api_base or 'default'}\")\n typer.echo(f\" Metric: {metric}\")\n typer.echo()\n\n # 5) Run evaluation with unified config\n model_config = {\n \"llm_model\": llm_model,\n \"api_key\": api_key,\n \"api_base\": api_base\n }\n \n try:\n result = run_evaluation(\n input_file=input,\n metric=metric,\n output_file=output,\n output_format=format,\n verbose=verbose,\n model_config=model_config\n )\n \n if result:\n typer.echo(result)\n \n except Exception as e:\n error_msg = str(e)\n \n # Check for quota/rate limit errors and show friendly message\n if any(keyword in error_msg.lower() for keyword in [\"quota\", \"rate limit\", \"insufficient_quota\", \"billing\"]):\n typer.echo(f\"\\nError: API quota exceeded\")\n typer.echo(f\"Please check your billing and usage limits, or try again later.\")\n typer.echo(f\"Details: {error_msg}\")\n raise typer.Exit(1)\n elif \"authentication\" in error_msg.lower() or \"unauthorized\" in error_msg.lower():\n typer.echo(f\"\\nError: Authentication failed\")\n typer.echo(f\"Please check your API key and try again.\")\n typer.echo(f\"Details: {error_msg}\")\n raise typer.Exit(1)\n else:\n typer.echo(f\"\\nError: {error_msg}\")\n raise typer.Exit(1)\n\n@app.command()\ndef analyze(\n input: str = typer.Option(..., \"--input\", \"-i\", help=\"Path to input JSON file\"),\n metric: str = typer.Option(\"faithfulness\", \"--metric\", \"-m\", help=\"Analysis type: faithfulness, context_precision, or hallucination\"),\n llm_model: str = typer.Option(\"gpt-3.5-turbo\", \"--llm-model\", help=\"Model name for analysis\"),\n api_key: Optional[str] = typer.Option(None, \"--api-key\", help=\"API key for your chosen provider (or use .env)\"),\n api_base: Optional[str] = typer.Option(None, \"--api-base\", help=\"(Optional) Custom API base URL (or use .env)\"),\n verbose: bool = typer.Option(False, \"--verbose\", \"-v\", help=\"Enable verbose output\"),\n):\n \"\"\"\n Analyze RAG metrics with detailed breakdowns.\n \n Faithfulness: Shows claim-by-claim analysis\n Context Precision: Shows chunk-by-chunk relevance analysis with Precision@k\n Hallucination: Shows detailed hallucination detection analysis\n \"\"\"\n \n # Setup API keys (same logic as test command)\n if api_key:\n key_var = detect_provider(llm_model, api_base)\n os.environ[key_var] = api_key\n os.environ[\"OPENAI_API_KEY\"] = api_key\n else:\n key_var = detect_provider(llm_model, api_base)\n env_api_key = os.getenv(key_var)\n if not env_api_key:\n typer.echo(f\"Error: No API key found. Set {key_var} in .env file or use --api-key\")\n raise typer.Exit(1)\n api_key = env_api_key\n \n clean_model = clean_model_name(llm_model, api_base)\n os.environ[\"RAGCLI_LLM_MODEL\"] = clean_model\n \n if api_base:\n os.environ[\"OPENAI_API_BASE\"] = api_base\n \n if api_base and \"openrouter.ai\" in api_base and api_key.startswith(\"sk-or-v1-\"):\n os.environ[\"LITELLM_API_KEY\"] = api_key\n os.environ[\"LITELLM_API_BASE\"] = api_base\n os.environ[\"OPENROUTER_API_KEY\"] = api_key\n print(f\"Set LiteLLM vars for OpenRouter\")\n \n # Load input data\n from runner import load_input_data\n data_list = load_input_data(input)\n \n # Build model config for evaluator\n model_config = {\n \"llm_model\": llm_model,\n \"api_key\": api_key,\n \"api_base\": api_base\n }\n \n # Initialize appropriate evaluator based on metric\n if metric == \"faithfulness\":\n from evaluators.ragas_faithfulness import RagasFaithfulnessEvaluator\n evaluator = RagasFaithfulnessEvaluator(model_config)\n typer.echo(\"RAGAS Faithfulness - Detailed Claim Analysis\")\n typer.echo(\"=\" * 50)\n \n for i, data in enumerate(data_list, 1):\n typer.echo(f\"\\nTest Case {i}:\")\n typer.echo(\"-\" * 30)\n \n analysis = evaluator.get_detailed_analysis(data)\n \n typer.echo(f\"Question: {data.get('question', '')}\")\n typer.echo(f\"Context: {' '.join(data.get('context', []))}\")\n typer.echo(f\"Answer: {data.get('answer', '')}\")\n typer.echo()\n \n if \"error\" in analysis:\n typer.echo(f\"Error: {analysis['error']}\")\n continue\n \n # Display formula explanation\n if 'formula_explanation' in analysis:\n formula = analysis['formula_explanation']\n typer.echo(\"📊 Faithfulness Formula:\")\n typer.echo(f\" {formula['faithfulness_formula']}\")\n typer.echo(f\" Calculation: {formula['calculation']}\")\n typer.echo(f\" Interpretation: {formula['interpretation']}\")\n typer.echo()\n \n # Display step-by-step verification process\n if 'verification_steps' in analysis:\n typer.echo(\"🔍 Claim Analysis:\")\n for step in analysis['verification_steps']:\n status_icon = \"✓\" if step['is_supported'] else \"✗\"\n status_text = \"SUPPORTED\" if step['is_supported'] else \"NOT SUPPORTED\"\n typer.echo(f\" {status_icon} Claim {step['step_number']}: {step['claim']}\")\n typer.echo(f\" Status: {status_text}\")\n typer.echo(f\" Verification: {step['verification_response']}\")\n \n # Add brief context explanation\n if step['is_supported']:\n typer.echo(f\" Context: Found supporting evidence in retrieved chunks\")\n else:\n typer.echo(f\" Context: No supporting evidence found in retrieved chunks\")\n typer.echo()\n \n # Display enhanced context usage analysis\n if 'context_usage_analysis' in analysis and 'chunks' in analysis['context_usage_analysis']:\n chunks = analysis['context_usage_analysis']['chunks']\n typer.echo(\"Context Usage Analysis:\")\n for chunk in chunks:\n usage_icon = \"✓\" if chunk['usage_status'] != 'unused_or_irrelevant' else \"✗\"\n typer.echo(f\" {usage_icon} Chunk {chunk['chunk_number']}: {chunk['usage_status'].replace('_', ' ').title()}\")\n typer.echo(f\" Text: {chunk['chunk_text'][:80]}{'...' if len(chunk['chunk_text']) > 80 else ''}\")\n if chunk['supporting_claims']:\n typer.echo(f\" Supporting: {len(chunk['supporting_claims'])} claims\")\n typer.echo()\n \n # Display diagnostic insights\n if 'diagnostic_insights' in analysis:\n insights = analysis['diagnostic_insights']\n typer.echo(f\"Diagnostic Insights (Severity: {insights.get('severity', 'unknown').upper()}):\")\n if insights.get('primary_issues'):\n typer.echo(\" Issues Found:\")\n for issue in insights['primary_issues']:\n typer.echo(f\" • {issue}\")\n if insights.get('recommendations'):\n typer.echo(\" Recommendations:\")\n for rec in insights['recommendations']:\n typer.echo(f\" • {rec}\")\n if insights.get('specific_problems'):\n typer.echo(\" Specific Unsupported Claims:\")\n for problem in insights['specific_problems']:\n typer.echo(f\" • Claim {problem['claim_number']}: {problem['claim_text']}\")\n typer.echo()\n \n # Display summary verdict prominently for casual users\n if 'summary_verdict' in analysis:\n typer.echo(\"📋 Summary Verdict:\")\n typer.echo(f\" {analysis['summary_verdict']}\")\n typer.echo()\n \n # Display action recommendations\n if 'action_recommendations' in analysis:\n action_rec = analysis['action_recommendations']\n typer.echo(\"🎯 Action Recommendations:\")\n typer.echo(f\" Action: {action_rec.get('action', 'none').replace('_', ' ').title()}\")\n typer.echo(f\" Reason: {action_rec.get('reason', 'unknown').replace('_', ' ').title()}\")\n typer.echo(f\" Confidence: {action_rec.get('confidence', 'unknown').title()}\")\n if action_rec.get('specific_actions'):\n typer.echo(\" Specific Actions:\")\n for action in action_rec['specific_actions']:\n typer.echo(f\" • {action}\")\n typer.echo()\n \n typer.echo(f\"Summary:\")\n typer.echo(f\" Total Claims: {analysis['total_claims']}\")\n typer.echo(f\" Supported Claims: {analysis['supported_claims']}\")\n typer.echo(f\" Unsupported Claims: {analysis['unsupported_claims']}\")\n typer.echo(f\" Faithfulness Score: {analysis['faithfulness_score']:.3f}\")\n \n # Display additional metrics\n if 'summary_statistics' in analysis:\n stats = analysis['summary_statistics']\n typer.echo()\n typer.echo(f\"Additional Metrics:\")\n typer.echo(f\" Support Rate: {stats.get('support_rate', 0.0):.1%}\")\n typer.echo(f\" Unsupported Rate: {stats.get('unsupported_rate', 0.0):.1%}\")\n typer.echo(f\" Claim Density: {stats.get('claim_density', 0.0):.3f} claims/word\")\n typer.echo(f\" Average Claim Length: {stats.get('average_claim_length', 0.0):.1f} words\")\n \n # Display context usage summary\n if 'context_usage_analysis' in analysis and 'summary' in analysis['context_usage_analysis']:\n ctx_summary = analysis['context_usage_analysis']['summary']\n typer.echo()\n typer.echo(f\"Context Utilization:\")\n typer.echo(f\" Used Chunks: {ctx_summary.get('used_chunks', 0)}/{ctx_summary.get('total_chunks', 0)}\")\n typer.echo(f\" Utilization Rate: {ctx_summary.get('context_utilization_rate', 0.0):.1%}\")\n typer.echo(f\" Total Claims Supported: {ctx_summary.get('total_claims_supported', 0)}\")\n typer.echo(f\" Average Relevance Score: {ctx_summary.get('average_relevance_score', 0.0):.3f}\")\n \n typer.echo()\n \n elif metric == \"context_precision\":\n from evaluators.ragas_context_precision import RagasContextPrecisionEvaluator\n evaluator = RagasContextPrecisionEvaluator(model_config)\n typer.echo(\"RAGAS Context Precision - Detailed Chunk Analysis\")\n typer.echo(\"=\" * 50)\n \n for i, data in enumerate(data_list, 1):\n typer.echo(f\"\\nTest Case {i}:\")\n typer.echo(\"-\" * 30)\n \n analysis = evaluator.get_detailed_analysis(data)\n \n typer.echo(f\"Question: {data.get('question', '')}\")\n typer.echo(f\"Answer: {data.get('answer', '')}\")\n typer.echo(f\"Retrieved Contexts ({analysis.get('total_chunks', 0)} chunks):\")\n for j, context in enumerate(data.get('context', []), 1):\n typer.echo(f\" [{j}] {context}\")\n typer.echo()\n \n if \"error\" in analysis:\n typer.echo(f\"Error: {analysis['error']}\")\n continue\n \n typer.echo(\"Chunk Relevance Analysis:\")\n for chunk_data in analysis['chunk_analysis']:\n status = \"✓ RELEVANT\" if chunk_data['relevant'] else \"✗ NOT RELEVANT\"\n typer.echo(f\" Chunk {chunk_data['chunk_number']}: {status}\")\n typer.echo(f\" Text: {chunk_data['chunk_text'][:80]}{'...' if len(chunk_data['chunk_text']) > 80 else ''}\")\n \n typer.echo()\n typer.echo(\"Precision@k Calculation:\")\n for precision_data in analysis['precision_at_k']:\n typer.echo(f\" Precision@{precision_data['position']}: {precision_data['true_positives']}/{precision_data['total_chunks']} = {precision_data['precision']}\")\n \n typer.echo()\n \n # Display retrieval quality analysis\n if 'retrieval_quality_analysis' in analysis:\n quality = analysis['retrieval_quality_analysis']\n typer.echo(\"Retrieval Quality Analysis:\")\n typer.echo(f\" Precision Trend: {quality.get('precision_trend', 'unknown').replace('_', ' ').title()}\")\n typer.echo(f\" Ranking Quality: {quality.get('ranking_quality', 'unknown').replace('_', ' ').title()}\")\n typer.echo(f\" Noise Pattern: {quality.get('noise_pattern', 'unknown').replace('_', ' ').title()}\")\n typer.echo()\n \n # Display simplified diagnostic insights\n if 'diagnostic_insights' in analysis:\n insights = analysis['diagnostic_insights']\n severity = insights.get('severity', 'unknown').upper()\n if severity in ['HIGH', 'CRITICAL']:\n typer.echo(f\"⚠️ {severity} RISK: {insights.get('primary_issues', ['Hallucination detected'])[0]}\")\n elif severity == 'LOW':\n typer.echo(\"✅ Low risk - minor issues detected\")\n \n # Display summary verdict prominently for casual users\n if 'summary_verdict' in analysis:\n typer.echo(\"📋 Summary Verdict:\")\n typer.echo(f\" {analysis['summary_verdict']}\")\n typer.echo()\n \n typer.echo(f\"Summary:\")\n typer.echo(f\" Total Chunks: {analysis['total_chunks']}\")\n typer.echo(f\" Relevant Chunks: {analysis['relevant_chunks']}\")\n typer.echo()\n \n # Display both precision metrics with clear explanations\n ragas_precision = analysis.get('ragas_context_precision', analysis.get('context_precision', 0.0))\n simple_precision = analysis.get('simple_context_precision', 0.0)\n \n typer.echo(f\"Context Precision Metrics:\")\n typer.echo(f\" 📊 RAGAS Context Precision: {ragas_precision}\")\n typer.echo(f\" → Measures ranking quality: 'Did we rank relevant chunks early?'\")\n typer.echo(f\" → Formula: Σ(Precision@k × v_k) / Σ(v_k)\")\n \n # Show detailed calculation if precision_at_k is available\n if 'precision_at_k' in analysis:\n typer.echo()\n typer.echo(\"Detailed RAGAS Calculation:\")\n precision_data = analysis['precision_at_k']\n relevant_chunks = analysis.get('relevant_chunks', 0)\n \n # Show Precision@k calculations\n typer.echo(\" Precision@k Calculations:\")\n for p_data in precision_data:\n k = p_data['position']\n precision = p_data['precision']\n true_pos = p_data['true_positives']\n total = p_data['total_chunks']\n typer.echo(f\" Precision@{k}: {true_pos}/{total} = {precision}\")\n \n # Show weighted calculation\n typer.echo()\n typer.echo(\" Weighted Calculation (v_k = 1 for relevant chunks):\")\n weighted_sum = 0\n for i, chunk in enumerate(analysis.get('chunk_analysis', [])):\n if chunk['relevant']:\n k = i + 1\n precision_k = precision_data[i]['precision']\n weighted_sum += precision_k\n typer.echo(f\" Position {k} (relevant): Precision@{k} × 1 = {precision_k}\")\n \n if relevant_chunks > 0:\n final_score = weighted_sum / relevant_chunks\n typer.echo(f\" Final Score: {weighted_sum} / {relevant_chunks} = {final_score:.3f}\")\n else:\n typer.echo(f\" Final Score: 0 (no relevant chunks)\")\n \n typer.echo()\n typer.echo(f\" 🔍 Simple Context Precision: {simple_precision}\")\n typer.echo(f\" → Measures retrieval cleanliness: 'How much retrieved content is useful?'\")\n typer.echo(f\" → Formula: Relevant Chunks / Total Chunks\")\n \n # Display summary statistics\n if 'summary_statistics' in analysis:\n stats = analysis['summary_statistics']\n typer.echo()\n typer.echo(f\"Additional Metrics:\")\n typer.echo(f\" Noise Rate: {stats.get('noise_rate', 0.0):.1%}\")\n typer.echo(f\" → Percentage of irrelevant chunks in retrieval\")\n typer.echo(f\" Early Precision: {stats.get('early_precision', 0.0):.3f}\")\n typer.echo(f\" → Precision at first 2 positions (ranking quality)\")\n typer.echo(f\" Late Precision: {stats.get('late_precision', 0.0):.3f}\")\n typer.echo(f\" → Precision at last 2 positions (overall noise)\")\n \n # Interpretive guidance based on both metrics\n typer.echo()\n typer.echo(\"Interpretation:\")\n if ragas_precision >= 0.8 and simple_precision >= 0.8:\n typer.echo(\" ✅ Excellent: Good ranking AND low noise\")\n typer.echo(\" → Relevant chunks appear early with minimal irrelevant content\")\n elif ragas_precision >= 0.8 and simple_precision < 0.5:\n typer.echo(\" ⚠️ Good early ranking but noisy retrieval\")\n typer.echo(\" → Relevant evidence appears first, but overall retrieval contains significant noise\")\n typer.echo(\" → Recommendation: Keep top results, prune or rerank the rest to suppress noise\")\n elif ragas_precision < 0.5 and simple_precision >= 0.8:\n typer.echo(\" ⚠️ Clean retrieval but poor ranking\")\n typer.echo(\" → Low noise but relevant chunks appear late in ranking\")\n typer.echo(\" → Recommendation: Improve ranking algorithm - top results should be most relevant\")\n else:\n typer.echo(\" ❌ Both ranking and noise need improvement\")\n typer.echo(\" → Poor early precision AND high noise content\")\n typer.echo(\" → Recommendation: Overhaul both similarity matching and ranking algorithms\")\n \n # Display action recommendations if available\n if 'action_recommendations' in analysis:\n action_rec = analysis['action_recommendations']\n typer.echo()\n typer.echo(\"🎯 Action Recommendations:\")\n \n action = action_rec.get('action', 'none')\n if action == 'maintain':\n typer.echo(f\" Action: ✅ Keep current retrieval system\")\n typer.echo(f\" → Your retrieval is working well\")\n elif action == 'overhaul_retrieval':\n typer.echo(f\" Action: 🔧 Fix both ranking and noise issues\")\n typer.echo(f\" → Improve similarity matching and ranking algorithms\")\n elif action == 'improve_ranking':\n typer.echo(f\" Action: 📈 Improve ranking algorithm\")\n typer.echo(f\" → Focus on putting most relevant chunks first\")\n elif action == 'reduce_noise':\n typer.echo(f\" Action: 🧹 Clean up retrieval noise\")\n typer.echo(f\" → Filter out irrelevant content\")\n else:\n typer.echo(f\" Action: {action}\")\n \n if action_rec.get('target_ranks'):\n ranks = action_rec['target_ranks']\n typer.echo(f\" Focus Positions: {ranks}\")\n typer.echo(f\" → These positions need improvement\")\n \n if action_rec.get('keep_ranks'):\n ranks = action_rec['keep_ranks']\n typer.echo(f\" Keep Positions: {ranks}\")\n typer.echo(f\" → These positions are working well\")\n \n reason = action_rec.get('reason', 'unknown')\n if reason == 'good_performance':\n typer.echo(f\" Reason: ✅ Good overall performance\")\n elif reason == 'poor_ranking_and_high_noise':\n typer.echo(f\" Reason: ❌ Poor ranking AND high noise\")\n elif reason == 'good_ranking_high_noise':\n typer.echo(f\" Reason: ⚠️ Good ranking but high noise\")\n elif reason == 'poor_ranking_low_noise':\n typer.echo(f\" Reason: ⚠️ Poor ranking but low noise\")\n else:\n typer.echo(f\" Reason: {reason}\")\n \n confidence = action_rec.get('confidence', 'unknown')\n if confidence == 'high':\n typer.echo(f\" Confidence: 🔴 High confidence in recommendation\")\n elif confidence == 'medium':\n typer.echo(f\" Confidence: 🟡 Medium confidence in recommendation\")\n elif confidence == 'low':\n typer.echo(f\" Confidence: 🟢 Low confidence in recommendation\")\n else:\n typer.echo(f\" Confidence: {confidence}\")\n \n typer.echo()\n \n elif metric == \"hallucination\":\n from evaluators.ragchecker_hallucination import RAGCheckerHallucinationEvaluator\n evaluator = RAGCheckerHallucinationEvaluator(model_config)\n typer.echo(\"RAGChecker Hallucination - Detailed Analysis\")\n typer.echo(\"=\" * 50)\n \n for i, data in enumerate(data_list, 1):\n typer.echo(f\"\\nTest Case {i}:\")\n typer.echo(\"-\" * 30)\n \n analysis = evaluator.get_detailed_analysis(data)\n \n typer.echo(f\"Question: {data.get('question', '')}\")\n typer.echo(f\"Answer: {data.get('answer', '')}\")\n typer.echo(f\"Context: {' '.join(data.get('context', []))}\")\n if data.get('ground_truth'):\n typer.echo(f\"Ground Truth: {data.get('ground_truth')}\")\n typer.echo()\n \n if \"error\" in analysis:\n typer.echo(f\"Error: {analysis['error']}\")\n continue\n \n # Display simplified formula explanation\n if 'formula_explanation' in analysis:\n formula = analysis['formula_explanation']\n typer.echo(\"Hallucination Score:\")\n typer.echo(f\" {formula['calculation']} = {formula['interpretation']}\")\n typer.echo()\n \n # Display enhanced claim-level analysis\n if 'claim_analysis' in analysis and 'claims' in analysis['claim_analysis']:\n claims = analysis['claim_analysis']['claims']\n typer.echo(\"Claim-Level Analysis:\")\n for claim in claims:\n status_icon = \"✓\" if claim['classification'] == 'correct_grounded' else \"✗\"\n typer.echo(f\" {status_icon} Claim {claim['claim_number']}: {claim['claim_text']}\")\n typer.echo(f\" Classification: {claim['classification'].replace('_', ' ').title()}\")\n if claim['context_support']['supported']:\n typer.echo(f\" Context Support: ✓ {claim['context_support']['explanation']}\")\n else:\n typer.echo(f\" Context Support: ✗ {claim['context_support']['explanation']}\")\n if analysis.get('evaluation_mode') == 'full' and 'ground_truth_support' in claim:\n if claim['ground_truth_support']['supported']:\n typer.echo(f\" Ground Truth: ✓ {claim['ground_truth_support']['explanation']}\")\n else:\n typer.echo(f\" Ground Truth: ✗ {claim['ground_truth_support']['explanation']}\")\n typer.echo()\n \n # Display context analysis\n if 'context_analysis' in analysis and 'chunks' in analysis['context_analysis']:\n chunks = analysis['context_analysis']['chunks']\n typer.echo(\"Context Usage Analysis:\")\n for chunk in chunks:\n usage_icon = \"✓\" if chunk['usage_analysis'] != 'irrelevant_noise' else \"✗\"\n typer.echo(f\" {usage_icon} Chunk {chunk['chunk_number']}: {chunk['usage_analysis'].replace('_', ' ').title()}\")\n typer.echo(f\" Text: {chunk['chunk_text'][:80]}{'...' if len(chunk['chunk_text']) > 80 else ''}\")\n if chunk['supports_claims']:\n claim_nums = [str(c['claim_number']) for c in chunk['supports_claims']]\n typer.echo(f\" Supports Claims: {', '.join(claim_nums)}\")\n typer.echo()\n \n # Display diagnostic insights\n if 'diagnostic_insights' in analysis:\n insights = analysis['diagnostic_insights']\n typer.echo(f\"Diagnostic Insights (Severity: {insights.get('severity', 'unknown').upper()}):\")\n if insights.get('primary_issues'):\n typer.echo(\" Issues Found:\")\n for issue in insights['primary_issues']:\n typer.echo(f\" • {issue}\")\n if insights.get('recommendations'):\n typer.echo(\" Recommendations:\")\n for rec in insights['recommendations']:\n typer.echo(f\" • {rec}\")\n if insights.get('specific_problems'):\n typer.echo(\" Irrelevant Chunks:\")\n for problem in insights['specific_problems']:\n typer.echo(f\" • Chunk {problem['chunk_number']}: {problem['chunk_text']}\")\n typer.echo()\n \n # Display summary verdict prominently for casual users\n if 'summary_verdict' in analysis:\n typer.echo(\"Summary Verdict:\")\n typer.echo(f\" {analysis['summary_verdict']}\")\n typer.echo()\n \n # Display action recommendations\n if 'action_recommendations' in analysis:\n action_rec = analysis['action_recommendations']\n typer.echo(\"Action Recommendations:\")\n typer.echo(f\" Action: {action_rec.get('action', 'none').replace('_', ' ').title()}\")\n typer.echo(f\" Reason: {action_rec.get('reason', 'unknown').replace('_', ' ').title()}\")\n typer.echo(f\" Confidence: {action_rec.get('confidence', 'unknown').title()}\")\n if action_rec.get('specific_actions'):\n typer.echo(\" Specific Actions:\")\n for action in action_rec['specific_actions']:\n typer.echo(f\" • {action}\")\n typer.echo()\n \n typer.echo(f\"Summary:\")\n typer.echo(f\" Total Claims: {analysis.get('summary_statistics', {}).get('total_claims', 0)}\")\n typer.echo(f\" Hallucinated Claims: {analysis.get('summary_statistics', {}).get('hallucinated_claims', 0)}\")\n typer.echo(f\" Correct Grounded Claims: {analysis.get('summary_statistics', {}).get('correct_grounded_claims', 0)}\")\n typer.echo(f\" Context Supported but Incorrect: {analysis.get('summary_statistics', {}).get('context_supported_but_incorrect', 0)}\")\n typer.echo(f\" Missing Context Evidence: {analysis.get('summary_statistics', {}).get('missing_context_evidence', 0)}\")\n typer.echo(f\" Hallucination Score: {analysis['hallucination_score']:.1f}%\")\n \n # Display additional metrics\n if 'summary_statistics' in analysis:\n stats = analysis['summary_statistics']\n typer.echo()\n typer.echo(f\"Additional Metrics:\")\n typer.echo(f\" Hallucination Rate: {stats.get('hallucination_rate', 0.0):.1%}\")\n typer.echo(f\" Grounding Rate: {stats.get('grounding_rate', 0.0):.1%}\")\n typer.echo(f\" Context Support Rate: {stats.get('context_support_rate', 0.0):.1%}\")\n typer.echo(f\" Claim Density: {stats.get('claim_density', 0.0):.3f} claims/word\")\n typer.echo(f\" Average Claim Length: {stats.get('average_claim_length', 0.0):.1f} words\")\n \n # Display context analysis summary\n if 'context_analysis' in analysis and 'summary' in analysis['context_analysis']:\n ctx_summary = analysis['context_analysis']['summary']\n typer.echo()\n typer.echo(f\"Context Analysis:\")\n typer.echo(f\" Total Chunks: {ctx_summary.get('total_chunks', 0)}\")\n typer.echo(f\" Relevant Chunks: {ctx_summary.get('relevant_chunks', 0)}\")\n typer.echo(f\" Context Precision: {ctx_summary.get('context_precision', 0.0):.3f}\")\n typer.echo(f\" Noise Chunks: {ctx_summary.get('noise_chunks', 0)}\")\n typer.echo(f\" Average Relevance Score: {ctx_summary.get('average_relevance_score', 0.0):.3f}\")\n \n typer.echo()\n typer.echo()\n \n else:\n typer.echo(f\"Error: Unknown metric '{metric}'. Available: faithfulness, context_precision, hallucination\")\n raise typer.Exit(1)\n\n@app.command()\ndef list_metrics():\n \"\"\"List available evaluation metrics.\"\"\"\n from evaluators import get_available_metrics\n \n typer.echo(\"Available Metrics:\")\n metrics = get_available_metrics()\n for name, description in metrics.items():\n typer.echo(f\" • {name}: {description}\")\n\n@app.command() \ndef version():\n \"\"\"Show version information.\"\"\"\n typer.echo(\"RAGTesterCLI - Unified RAG Evaluation\")\n typer.echo(\" Multi-provider LLM support (OpenAI, Anthropic, Google, OpenRouter)\")\n typer.echo(\" True RAGAS & RAGChecker integration\")\n typer.echo(\" One API key for any provider\")\n\nif __name__ == \"__main__\":\n app() "}}}} |