1 line
8.9 KiB
JSON
1 line
8.9 KiB
JSON
{"repo": "adam-bouafia/lct-llms-comparative-tool", "files": {"app/src/tools/model_compatibility_checker.py": {"284ee306df022ec0509648379df9c8f2aa9547c2": {"t": 1785674654.9855676, "n": 8346, "h": "d89a37156b823dc9", "c": "#!/usr/bin/env python3\n\"\"\"\nEnhanced Model Compatibility Checker for LLM Evaluation System\n\nThis utility helps users verify model compatibility before running experiments\nand provides recommendations for different use cases.\n\nSupports all HuggingFace Transformers models including popular architectures:\n- LLaMA 2 (Meta AI)\n- Mistral AI models\n- Google Gemma\n- Alibaba Qwen/Qwen2\n- Microsoft Phi models\n- 01-ai Yi models\n- And thousands more...\n\nUsage:\n python model_compatibility_checker.py model_name\n python model_compatibility_checker.py --list-recommended\n\"\"\"\n\nimport sys\nimport argparse\nfrom typing import Dict, List, Tuple, Optional\nfrom llm_runner.loaders.universal_loader import UniversalModelLoader\n\n\ndef get_recommended_models() -> Dict[str, List[Tuple[str, str]]]:\n \"\"\"Get recommended models by category with descriptions.\"\"\"\n return {\n \"small_fast\": [\n (\"gpt2\", \"Classic GPT-2: Fast, lightweight, great for testing\"),\n (\"distilgpt2\", \"Distilled GPT-2: Even faster, good baseline\"),\n (\"microsoft/phi-2\", \"Phi-2: Small but capable Microsoft model\"),\n (\"google/gemma-2b\", \"Gemma-2B: Google's efficient small model\"),\n (\"t5-small\", \"T5-Small: Versatile seq2seq model for many tasks\"),\n (\"google/flan-t5-small\", \"FLAN-T5-Small: Instruction-tuned T5\"),\n ],\n \"medium_performance\": [\n (\"meta-llama/Llama-2-7b-hf\", \"LLaMA 2-7B: High-quality Meta model\"),\n (\n \"mistralai/Mistral-7B-v0.1\",\n \"Mistral-7B: Efficient high-performance model\",\n ),\n (\"google/gemma-7b\", \"Gemma-7B: Google's balanced performance model\"),\n (\"Qwen/Qwen-7B\", \"Qwen-7B: Alibaba's multilingual model\"),\n (\n \"microsoft/Phi-3-mini-4k-instruct\",\n \"Phi-3: Microsoft's instruction-tuned model\",\n ),\n (\"01-ai/Yi-6B\", \"Yi-6B: High-quality Chinese-English model\"),\n ],\n \"code_models\": [\n (\"codellama/CodeLlama-7b-hf\", \"Code Llama: Meta's specialized code model\"),\n (\n \"Salesforce/codegen-350M-mono\",\n \"CodeGen: Salesforce code generation model\",\n ),\n (\"Salesforce/codet5-small\", \"CodeT5: Code-focused T5 variant\"),\n (\"microsoft/CodeGPT-small-py\", \"CodeGPT: Microsoft Python code model\"),\n ],\n \"chat_models\": [\n (\"microsoft/DialoGPT-medium\", \"DialoGPT: Conversational Microsoft model\"),\n (\"facebook/blenderbot-400M-distill\", \"BlenderBot: Facebook chat model\"),\n (\"microsoft/GODEL-v1_1-base-seq2seq\", \"GODEL: Task-oriented dialog model\"),\n ],\n \"summarization\": [\n (\"google/pegasus-xsum\", \"Pegasus XSum: Abstractive summarization\"),\n (\"google/pegasus-cnn_dailymail\", \"Pegasus CNN: News summarization\"),\n (\"facebook/bart-large\", \"BART-Large: Strong seq2seq model\"),\n (\"google/flan-t5-base\", \"FLAN-T5-Base: Instruction-tuned for tasks\"),\n ],\n \"translation\": [\n (\"Helsinki-NLP/opus-mt-en-de\", \"Marian: English-German translation\"),\n (\"facebook/mbart-large-50\", \"mBART: Multilingual translation model\"),\n (\"google/ul2\", \"UL2: Universal language learner\"),\n ],\n \"large_models\": [\n (\"EleutherAI/gpt-j-6b\", \"GPT-J: Large open-source model\"),\n (\"facebook/opt-1.3b\", \"OPT-1.3B: Facebook's large model\"),\n (\"bigscience/bloom-1b7\", \"BLOOM: Multilingual large model\"),\n (\"tiiuae/falcon-7b\", \"Falcon-7B: High-performance UAE model\"),\n ],\n }\n\n\ndef check_model_compatibility(model_id: str) -> Dict:\n \"\"\"Check compatibility of a specific model.\"\"\"\n print(f\"\\n🔍 CHECKING COMPATIBILITY: {model_id}\")\n print(\"=\" * 50)\n\n try:\n loader = UniversalModelLoader()\n\n # Create metadata\n class TestMetadata:\n def __init__(self, model_id):\n self.id = model_id\n self.model_id = model_id\n self.pipeline_tag = None\n self.task = None\n\n metadata = TestMetadata(model_id)\n\n # Test model class detection\n model_class = loader._get_model_class(metadata)\n pipeline_task = loader._get_pipeline_task(metadata)\n\n print(f\"✅ Model ID: {model_id}\")\n print(f\"✅ Detected Class: {model_class.__name__}\")\n print(f\"✅ Pipeline Task: {pipeline_task}\")\n\n # Determine model category\n if \"Seq2Seq\" in model_class.__name__:\n category = \"Sequence-to-Sequence (Encoder-Decoder)\"\n use_case = \"Translation, summarization, text-to-text tasks\"\n else:\n category = \"Causal Language Model (Decoder-only)\"\n use_case = \"Text generation, completion, chat\"\n\n print(f\"✅ Category: {category}\")\n print(f\"✅ Best Use Case: {use_case}\")\n\n # Check for potential issues\n warnings = []\n\n if (\n any(\n pattern in model_id.lower()\n for pattern in [\"llama\", \"mistral\", \"falcon\"]\n )\n and \"7b\" in model_id.lower()\n ):\n warnings.append(\"⚠️ Large model (7B+) - ensure sufficient RAM/VRAM\")\n\n if \"instruct\" in model_id.lower() or \"chat\" in model_id.lower():\n warnings.append(\n \"💡 Instruction-tuned model - may work better with specific prompts\"\n )\n\n if any(pattern in model_id.lower() for pattern in [\"code\", \"python\", \"java\"]):\n warnings.append(\n \"💻 Code-specialized model - optimized for programming tasks\"\n )\n\n return {\n \"compatible\": True,\n \"model_class\": model_class.__name__,\n \"pipeline_task\": pipeline_task,\n \"category\": category,\n \"use_case\": use_case,\n \"warnings\": warnings,\n }\n\n except Exception as e:\n print(f\"❌ ERROR: {str(e)}\")\n return {\n \"compatible\": False,\n \"error\": str(e),\n \"suggestions\": [\n \"Check if model ID is correct\",\n \"Verify model exists on HuggingFace Hub\",\n \"Try adding trust_remote_code=True if using custom code\",\n ],\n }\n\n\ndef list_recommended_models():\n \"\"\"Display all recommended models by category.\"\"\"\n print(\"\\n🎯 RECOMMENDED MODELS BY CATEGORY\")\n print(\"=\" * 50)\n\n models = get_recommended_models()\n\n for category, model_list in models.items():\n print(f\"\\n📂 {category.upper().replace('_', ' ')}\")\n print(\"-\" * 30)\n\n for model_id, description in model_list:\n print(f\" • {model_id}\")\n print(f\" {description}\")\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Check model compatibility for LLM evaluation system\"\n )\n parser.add_argument(\n \"model\",\n nargs=\"?\",\n help=\"Model ID to check (e.g., 'gpt2', 'meta-llama/Llama-2-7b-hf')\",\n )\n parser.add_argument(\n \"--list-recommended\",\n action=\"store_true\",\n help=\"List all recommended models by category\",\n )\n\n args = parser.parse_args()\n\n if args.list_recommended:\n list_recommended_models()\n elif args.model:\n result = check_model_compatibility(args.model)\n\n if result[\"compatible\"]:\n print(f\"\\n✅ COMPATIBILITY: EXCELLENT\")\n print(f\" Ready for LLM evaluation experiments!\")\n\n if result.get(\"warnings\"):\n print(f\"\\n⚠️ NOTES:\")\n for warning in result[\"warnings\"]:\n print(f\" {warning}\")\n else:\n print(f\"\\n❌ COMPATIBILITY: ISSUES DETECTED\")\n print(f\" Error: {result.get('error', 'Unknown error')}\")\n\n if result.get(\"suggestions\"):\n print(f\"\\n💡 SUGGESTIONS:\")\n for suggestion in result[\"suggestions\"]:\n print(f\" • {suggestion}\")\n else:\n parser.print_help()\n print(f\"\\n💡 EXAMPLES:\")\n print(f\" python model_compatibility_checker.py gpt2\")\n print(f\" python model_compatibility_checker.py meta-llama/Llama-2-7b-hf\")\n print(f\" python model_compatibility_checker.py --list-recommended\")\n\n\nif __name__ == \"__main__\":\n main()\n"}}}} |