{"repo": "abhimittal1/magicpin", "files": {"judge_simulator.py": {"b728ad4aa16b1b6c136a6b46c179c693cc225914": {"t": 1785746395.4649677, "n": 36150, "h": "a9911e1d38eabeab", "c": "#!/usr/bin/env python3\n\"\"\"\nmagicpin AI Challenge — LLM-Powered Judge Simulator\n====================================================\n\nA strict but fair judge that scores your bot and explains WHY.\n\nHOW TO USE:\n1. Edit the CONFIGURATION section below (lines 25-45)\n2. Set your LLM provider and API key\n3. Set your bot URL\n4. Run: python judge_simulator.py\n\nThat's it!\n\nAuthor: magicpin AI Challenge Team\n\"\"\"\n\n# =============================================================================\n# ██████  CONFIGURATION - EDIT THIS SECTION ██████\n# =============================================================================\n\n# Your bot's URL (where your bot is running)\nBOT_URL = \"http://localhost:8081\"\n\n# Choose your LLM provider: \"openai\", \"anthropic\", \"gemini\", \"deepseek\", \"groq\", \"ollama\", \"openrouter\"\nLLM_PROVIDER = \"openai\"\n\n# Your API key (paste your key here)\n\n# Model to use (leave empty for default, or specify like \"gpt-4o\", \"claude-3-5-sonnet-20241022\", etc.)\nLLM_MODEL = \"gpt-4.1-mini\"  # <-- Optional: specify model or leave empty for default\n\n# For Ollama only: local server URL\nOLLAMA_URL = \"http://localhost:11434\"\n\n# Which test to run by default\nTEST_SCENARIO = \"all\"\n\n# =============================================================================\n# ██████  END OF CONFIGURATION - DON'T EDIT BELOW THIS LINE ██████\n# =============================================================================\n\nimport os\nimport sys\nimport json\nimport time\nimport re\nimport socket\nfrom datetime import datetime\nfrom dataclasses import dataclass, field\nfrom typing import Optional, List, Dict, Any, Tuple\nfrom pathlib import Path\nfrom urllib import request as urlrequest, error as urlerror\nfrom abc import ABC, abstractmethod\n\n# Constants\nTIMEOUT_LLM = 45\nDATASET_DIR = Path(__file__).parent / \"dataset\"\n\n# =============================================================================\n# TERMINAL OUTPUT\n# =============================================================================\n\nclass Colors:\n    HEADER = '\\033[95m'\n    BLUE = '\\033[94m'\n    CYAN = '\\033[96m'\n    GREEN = '\\033[92m'\n    YELLOW = '\\033[93m'\n    RED = '\\033[91m'\n    MAGENTA = '\\033[35m'\n    BOLD = '\\033[1m'\n    DIM = '\\033[2m'\n    RESET = '\\033[0m'\n\ndef print_header(text: str):\n    print(f\"\\n{Colors.HEADER}{Colors.BOLD}{'='*70}{Colors.RESET}\")\n    print(f\"{Colors.HEADER}{Colors.BOLD}{text.center(70)}{Colors.RESET}\")\n    print(f\"{Colors.HEADER}{Colors.BOLD}{'='*70}{Colors.RESET}\\n\")\n\ndef print_section(text: str):\n    print(f\"\\n{Colors.CYAN}{Colors.BOLD}--- {text} ---{Colors.RESET}\\n\")\n\ndef print_success(text: str):\n    print(f\"{Colors.GREEN}[PASS]{Colors.RESET} {text}\")\n\ndef print_fail(text: str):\n    print(f\"{Colors.RED}[FAIL]{Colors.RESET} {text}\")\n\ndef print_warn(text: str):\n    print(f\"{Colors.YELLOW}[WARN]{Colors.RESET} {text}\")\n\ndef print_info(text: str):\n    print(f\"{Colors.BLUE}[INFO]{Colors.RESET} {text}\")\n\ndef print_llm(text: str):\n    print(f\"{Colors.MAGENTA}[LLM]{Colors.RESET} {text}\")\n\ndef print_score_bar(dimension: str, score: int, max_score: int = 10):\n    bar_filled = int((score / max_score) * 20)\n    bar_empty = 20 - bar_filled\n    color = Colors.GREEN if score >= 7 else Colors.YELLOW if score >= 4 else Colors.RED\n    print(f\"  {dimension:22} [{color}{'█' * bar_filled}{Colors.DIM}{'░' * bar_empty}{Colors.RESET}] {color}{score:2}/{max_score}{Colors.RESET}\")\n\ndef print_reason(text: str):\n    wrapped = text[:200] + \"...\" if len(text) > 200 else text\n    print(f\"    {Colors.DIM}{wrapped}{Colors.RESET}\")\n\ndef print_hint(hint: str):\n    print(f\"\\n  {Colors.YELLOW}Hint:{Colors.RESET} {hint}\")\n\n# =============================================================================\n# DATA CLASSES\n# =============================================================================\n\n@dataclass\nclass ScoreResult:\n    specificity: int = 0\n    specificity_reason: str = \"\"\n    category_fit: int = 0\n    category_fit_reason: str = \"\"\n    merchant_fit: int = 0\n    merchant_fit_reason: str = \"\"\n    decision_quality: int = 0\n    decision_quality_reason: str = \"\"\n    engagement_compulsion: int = 0\n    engagement_reason: str = \"\"\n    penalties: int = 0\n    penalty_reasons: List[str] = field(default_factory=list)\n    hint: str = \"\"\n\n    @property\n    def total(self) -> int:\n        return max(0, self.specificity + self.category_fit + self.merchant_fit +\n                   self.decision_quality + self.engagement_compulsion - self.penalties)\n\n# =============================================================================\n# LLM PROVIDERS\n# =============================================================================\n\nclass LLMProvider(ABC):\n    @abstractmethod\n    def complete(self, prompt: str, system: str = None) -> str:\n        pass\n\n    @abstractmethod\n    def name(self) -> str:\n        pass\n\n\nclass OpenAIProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"gpt-4o-mini\"\n\n    def name(self) -> str:\n        return f\"OpenAI ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        body = json.dumps({\n            \"model\": self.model,\n            \"messages\": messages,\n            \"temperature\": 0.2,\n            \"max_tokens\": 1500\n        }).encode(\"utf-8\")\n\n        req = urlrequest.Request(\n            \"https://api.openai.com/v1/chat/completions\",\n            data=body,\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\nclass AnthropicProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"claude-3-5-sonnet-20241022\"\n\n    def name(self) -> str:\n        return f\"Anthropic ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        body_dict = {\"model\": self.model, \"max_tokens\": 1500,\n                     \"messages\": [{\"role\": \"user\", \"content\": prompt}]}\n        if system:\n            body_dict[\"system\"] = system\n\n        req = urlrequest.Request(\n            \"https://api.anthropic.com/v1/messages\",\n            data=json.dumps(body_dict).encode(\"utf-8\"),\n            headers={\"x-api-key\": self.api_key, \"Content-Type\": \"application/json\",\n                     \"anthropic-version\": \"2023-06-01\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"content\"][0][\"text\"]\n\n\nclass GeminiProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"gemini-1.5-flash\"\n\n    def name(self) -> str:\n        return f\"Gemini ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        full_prompt = f\"{system}\\n\\n{prompt}\" if system else prompt\n        body = json.dumps({\n            \"contents\": [{\"parts\": [{\"text\": full_prompt}]}],\n            \"generationConfig\": {\"temperature\": 0.2, \"maxOutputTokens\": 1500}\n        }).encode(\"utf-8\")\n\n        url = f\"https://generativelanguage.googleapis.com/v1beta/models/{self.model}:generateContent?key={self.api_key}\"\n        req = urlrequest.Request(url, data=body, headers={\"Content-Type\": \"application/json\"})\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"candidates\"][0][\"content\"][\"parts\"][0][\"text\"]\n\n\nclass DeepSeekProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"deepseek-chat\"\n\n    def name(self) -> str:\n        return f\"DeepSeek ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        req = urlrequest.Request(\n            \"https://api.deepseek.com/v1/chat/completions\",\n            data=json.dumps({\"model\": self.model, \"messages\": messages,\n                            \"temperature\": 0.2, \"max_tokens\": 1500}).encode(\"utf-8\"),\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\nclass GroqProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"llama-3.1-70b-versatile\"\n\n    def name(self) -> str:\n        return f\"Groq ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        req = urlrequest.Request(\n            \"https://api.groq.com/openai/v1/chat/completions\",\n            data=json.dumps({\"model\": self.model, \"messages\": messages,\n                            \"temperature\": 0.2, \"max_tokens\": 1500}).encode(\"utf-8\"),\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\nclass OllamaProvider(LLMProvider):\n    def __init__(self, model: str = \"\", api_url: str = \"\"):\n        self.model = model or \"llama3\"\n        self.api_url = api_url or \"http://localhost:11434\"\n\n    def name(self) -> str:\n        return f\"Ollama ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        full_prompt = f\"{system}\\n\\n{prompt}\" if system else prompt\n        req = urlrequest.Request(\n            f\"{self.api_url}/api/generate\",\n            data=json.dumps({\"model\": self.model, \"prompt\": full_prompt,\n                            \"stream\": False, \"options\": {\"temperature\": 0.2}}).encode(\"utf-8\"),\n            headers={\"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=90)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"response\"]\n\n\nclass OpenRouterProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"anthropic/claude-3-haiku\"\n\n    def name(self) -> str:\n        return f\"OpenRouter ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        req = urlrequest.Request(\n            \"https://openrouter.ai/api/v1/chat/completions\",\n            data=json.dumps({\"model\": self.model, \"messages\": messages,\n                            \"temperature\": 0.2, \"max_tokens\": 1500}).encode(\"utf-8\"),\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\",\n                     \"HTTP-Referer\": \"https://magicpin.com\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef create_provider() -> LLMProvider:\n    \"\"\"Create LLM provider from configuration.\"\"\"\n    providers = {\n        \"openai\": lambda: OpenAIProvider(LLM_API_KEY, LLM_MODEL),\n        \"anthropic\": lambda: AnthropicProvider(LLM_API_KEY, LLM_MODEL),\n        \"gemini\": lambda: GeminiProvider(LLM_API_KEY, LLM_MODEL),\n        \"deepseek\": lambda: DeepSeekProvider(LLM_API_KEY, LLM_MODEL),\n        \"groq\": lambda: GroqProvider(LLM_API_KEY, LLM_MODEL),\n        \"ollama\": lambda: OllamaProvider(LLM_MODEL, OLLAMA_URL),\n        \"openrouter\": lambda: OpenRouterProvider(LLM_API_KEY, LLM_MODEL),\n    }\n\n    if LLM_PROVIDER not in providers:\n        print_fail(f\"Unknown provider: {LLM_PROVIDER}\")\n        print_info(f\"Available: {', '.join(providers.keys())}\")\n        sys.exit(1)\n\n    return providers[LLM_PROVIDER]()\n\n# =============================================================================\n# DATASET & BOT CLIENT\n# =============================================================================\n\nclass DatasetLoader:\n    def __init__(self, dataset_dir: Path):\n        self.dataset_dir = dataset_dir\n        self.categories = {}\n        self.merchants = {}\n        self.customers = {}\n        self.triggers = {}\n\n    def load(self) -> bool:\n        try:\n            cat_dir = self.dataset_dir / \"categories\"\n            if cat_dir.exists():\n                for f in cat_dir.glob(\"*.json\"):\n                    data = json.load(open(f))\n                    self.categories[data.get(\"slug\", f.stem)] = data\n\n            for name, container, key in [\n                (\"merchants_seed.json\", \"merchants\", \"merchant_id\"),\n                (\"customers_seed.json\", \"customers\", \"customer_id\"),\n                (\"triggers_seed.json\", \"triggers\", \"id\")\n            ]:\n                path = self.dataset_dir / name\n                if path.exists():\n                    data = json.load(open(path))\n                    items = data.get(container, data.get(container.rstrip(\"s\"), []))\n                    storage = getattr(self, container)\n                    for item in items:\n                        if key in item:\n                            storage[item[key]] = item\n            return True\n        except Exception as e:\n            print_fail(f\"Dataset load error: {e}\")\n            return False\n\n\nclass BotClient:\n    def __init__(self, base_url: str):\n        self.base_url = base_url.rstrip(\"/\")\n\n    def _request(self, method: str, path: str, timeout: int = 30,\n                 body_dict: Dict = None) -> Tuple[Optional[Dict], Optional[str], float]:\n        url = f\"{self.base_url}{path}\"\n        start = time.time()\n        body = json.dumps(body_dict).encode(\"utf-8\") if body_dict else None\n        headers = {\"Content-Type\": \"application/json\"}\n        req = urlrequest.Request(url, data=body, method=method, headers=headers)\n\n        try:\n            resp = urlrequest.urlopen(req, timeout=timeout)\n            return json.loads(resp.read().decode(\"utf-8\")), None, (time.time() - start) * 1000\n        except urlerror.HTTPError as e:\n            latency = (time.time() - start) * 1000\n            if e.code == 401:\n                return None, \"Unauthorized\", latency\n            try:\n                return json.loads(e.read().decode(\"utf-8\")), None, latency\n            except:\n                return None, f\"HTTP {e.code}\", latency\n        except Exception as e:\n            return None, str(e), (time.time() - start) * 1000\n\n    def healthz(self):\n        return self._request(\"GET\", \"/v1/healthz\", 5)\n\n    def metadata(self):\n        return self._request(\"GET\", \"/v1/metadata\", 5)\n\n    def push_context(self, scope, cid, version, payload):\n        return self._request(\"POST\", \"/v1/context\", 10, {\n            \"scope\": scope, \"context_id\": cid, \"version\": version,\n            \"payload\": payload, \"delivered_at\": datetime.utcnow().isoformat() + \"Z\"\n        })\n\n    def tick(self, triggers):\n        return self._request(\"POST\", \"/v1/tick\", 15, {\n            \"now\": datetime.utcnow().isoformat() + \"Z\", \"available_triggers\": triggers\n        })\n\n    def reply(self, conv_id, merchant_id, message, turn):\n        return self._request(\"POST\", \"/v1/reply\", 15, {\n            \"conversation_id\": conv_id, \"merchant_id\": merchant_id, \"customer_id\": None,\n            \"from_role\": \"merchant\", \"message\": message,\n            \"received_at\": datetime.utcnow().isoformat() + \"Z\", \"turn_number\": turn\n        })\n\n# =============================================================================\n# LLM SCORING ENGINE\n# =============================================================================\n\nclass LLMScorer:\n    \"\"\"Scores messages using LLM and provides detailed reasoning.\"\"\"\n\n    SYSTEM = \"\"\"You are a STRICT judge for the magicpin AI Challenge. You score merchant engagement messages.\n\nSCORING DIMENSIONS (0-10 each, be strict - 5 is average, 7+ is good, 9+ is excellent):\n\n1. SPECIFICITY: Does the message have VERIFIABLE facts?\n   - Numbers (percentages, counts, prices)\n   - Dates/times\n   - Source citations\n   - Concrete claims vs vague statements\n\n2. CATEGORY FIT: Does the voice match the business type?\n   - Dentists: clinical, peer-to-peer, technical OK, use \"Dr.\" prefix\n   - Salons: warm, friendly, practical\n   - Restaurants: operator-to-operator\n   - Gyms: coaching, motivational\n   - Pharmacies: trustworthy, precise\n\n3. MERCHANT FIT: Is it personalized to THIS merchant?\n   - Uses their name/owner name correctly\n   - References their actual data (not fabricated)\n   - Honors language preference\n\n4. TRIGGER RELEVANCE: Does it connect to WHY NOW?\n   - Clear reason for this specific message\n   - Uses data from the trigger payload\n   - Not a generic nudge\n\n5. ENGAGEMENT COMPULSION: Would they reply?\n   - Loss aversion, curiosity, social proof\n   - Clear CTA\n   - Low friction ask\n\nPENALTIES:\n- Fabricating data not in context: -2\n- Exposing internal jargon to merchant: -1\n\nRESPOND ONLY WITH THIS EXACT JSON FORMAT:\n{\n  \"specificity\": <0-10>,\n  \"specificity_reason\": \"<why this score, 1-2 sentences>\",\n  \"category_fit\": <0-10>,\n  \"category_fit_reason\": \"<why this score>\",\n  \"merchant_fit\": <0-10>,\n  \"merchant_fit_reason\": \"<why this score>\",\n  \"decision_quality\": <0-10>,\n  \"decision_quality_reason\": \"<why this score>\",\n  \"engagement_compulsion\": <0-10>,\n  \"engagement_reason\": \"<why this score>\",\n  \"hint\": \"<one sentence guidance for improvement, cryptic not direct>\"\n}\"\"\"\n\n    def __init__(self, llm: LLMProvider, dataset: DatasetLoader):\n        self.llm = llm\n        self.dataset = dataset\n\n    def score(self, action: Dict, category: Dict, merchant: Dict,\n              trigger: Dict, customer: Dict = None) -> ScoreResult:\n        \"\"\"Score a message and return detailed results.\"\"\"\n\n        body = action.get(\"body\", \"\")\n\n        prompt = f\"\"\"SCORE THIS MESSAGE:\n\n=== CONTEXT PROVIDED TO BOT ===\nCategory: {category.get('slug', 'unknown')}\nVoice: {category.get('voice', {}).get('tone', 'unknown')}\nTaboos: {category.get('voice', {}).get('vocab_taboo', [])[:5]}\n\nMerchant: {merchant.get('identity', {}).get('name', 'unknown')}\nOwner: {merchant.get('identity', {}).get('owner_first_name', 'unknown')}\nLocality: {merchant.get('identity', {}).get('locality', 'unknown')}\nLanguages: {merchant.get('identity', {}).get('languages', [])}\nPerformance: views={merchant.get('performance', {}).get('views', '?')}, calls={merchant.get('performance', {}).get('calls', '?')}, ctr={merchant.get('performance', {}).get('ctr', '?')}\nSignals: {merchant.get('signals', [])}\nActive Offers: {[o.get('title') for o in merchant.get('offers', []) if o.get('status') == 'active']}\n\nTrigger Kind: {trigger.get('kind', 'unknown')}\nTrigger Payload: {json.dumps(trigger.get('payload', {}))}\nTrigger Urgency: {trigger.get('urgency', '?')}\n\nCustomer: {json.dumps(customer.get('identity', {})) if customer else 'None (merchant-facing)'}\n\n=== BOT'S MESSAGE ===\nBody ({len(body)} chars): \"{body}\"\nCTA: {action.get('cta', 'none')}\nSend As: {action.get('send_as', 'vera')}\n\nScore each dimension 0-10 with clear reasoning. Be STRICT.\"\"\"\n\n        try:\n            print_llm(\"Analyzing message...\")\n            response = self.llm.complete(prompt, self.SYSTEM)\n            return self._parse_response(response, action)\n        except Exception as e:\n            print_warn(f\"LLM error: {e}\")\n            return self._fallback_score(action)\n\n    def _parse_response(self, response: str, action: Dict) -> ScoreResult:\n        \"\"\"Parse LLM JSON response.\"\"\"\n        match = re.search(r'\\{[\\s\\S]*\\}', response)\n        if not match:\n            return self._fallback_score(action)\n\n        try:\n            data = json.loads(match.group())\n            result = ScoreResult(\n                specificity=min(10, max(0, int(data.get(\"specificity\", 5)))),\n                specificity_reason=data.get(\"specificity_reason\", \"\"),\n                category_fit=min(10, max(0, int(data.get(\"category_fit\", 5)))),\n                category_fit_reason=data.get(\"category_fit_reason\", \"\"),\n                merchant_fit=min(10, max(0, int(data.get(\"merchant_fit\", 5)))),\n                merchant_fit_reason=data.get(\"merchant_fit_reason\", \"\"),\n                decision_quality=min(10, max(0, int(data.get(\"decision_quality\", data.get(\"trigger_relevance\", 5))))),\n                decision_quality_reason=data.get(\"decision_quality_reason\", data.get(\"trigger_relevance_reason\", \"\")),\n                engagement_compulsion=min(10, max(0, int(data.get(\"engagement_compulsion\", 5)))),\n                engagement_reason=data.get(\"engagement_reason\", \"\"),\n                hint=data.get(\"hint\", \"\")\n            )\n            return result\n        except Exception as e:\n            print_warn(f\"Parse error: {e}\")\n            return self._fallback_score(action)\n\n    def _fallback_score(self, action: Dict) -> ScoreResult:\n        \"\"\"Basic fallback scoring.\"\"\"\n        body = action.get(\"body\", \"\").lower()\n        nums = len(re.findall(r'\\d+', body))\n        return ScoreResult(\n            specificity=min(10, 3 + nums * 2),\n            specificity_reason=\"Fallback: counted numbers in message\",\n            category_fit=5, category_fit_reason=\"Could not evaluate\",\n            merchant_fit=5, merchant_fit_reason=\"Could not evaluate\",\n            decision_quality=5, decision_quality_reason=\"Could not evaluate\",\n            engagement_compulsion=5, engagement_reason=\"Could not evaluate\",\n            hint=\"LLM scoring failed - using basic heuristics\"\n        )\n\n# =============================================================================\n# MAIN JUDGE\n# =============================================================================\n\nclass JudgeSimulator:\n    def __init__(self, llm: LLMProvider):\n        self.llm = llm\n        self.client = BotClient(BOT_URL)\n        self.dataset = DatasetLoader(DATASET_DIR)\n        self.scorer: Optional[LLMScorer] = None\n        self.all_scores: List[ScoreResult] = []\n\n    def run(self, scenario: str) -> bool:\n        print_header(f\"LLM JUDGE — {scenario.upper()}\")\n        print_info(f\"Bot: {BOT_URL}\")\n        print_info(f\"LLM: {self.llm.name()}\")\n\n        if not self.dataset.load():\n            print_fail(\"Dataset load failed\")\n            return False\n\n        self.scorer = LLMScorer(self.llm, self.dataset)\n        print_info(f\"Loaded: {len(self.dataset.categories)} categories, \"\n                   f\"{len(self.dataset.merchants)} merchants, \"\n                   f\"{len(self.dataset.triggers)} triggers\")\n\n        scenarios = {\n            \"warmup\": self._warmup,\n            \"phase2_short\": self._phase2_short,\n            \"auto_reply_hell\": self._auto_reply,\n            \"intent_transition\": self._intent,\n            \"hostile\": self._hostile,\n            \"all\": self._all,\n            \"full_evaluation\": self._full,\n        }\n\n        if scenario not in scenarios:\n            print_fail(f\"Unknown scenario: {scenario}\")\n            print_info(f\"Available: {', '.join(scenarios.keys())}\")\n            return False\n\n        success = scenarios[scenario]()\n        self._final_summary()\n        return success\n\n    def _warmup(self) -> bool:\n        print_section(\"WARMUP\")\n\n        data, err, lat = self.client.healthz()\n        if err:\n            print_fail(f\"healthz: {err}\")\n            return False\n        print_success(f\"healthz ({lat:.0f}ms)\")\n\n        data, err, lat = self.client.metadata()\n        if err:\n            print_warn(f\"metadata: {err}\")\n        else:\n            print_success(f\"metadata — Team: {data.get('team_name', '?')}, Model: {data.get('model', '?')}\")\n\n        print_section(\"CONTEXT PUSH\")\n        for slug, cat in self.dataset.categories.items():\n            data, err, _ = self.client.push_context(\"category\", slug, 1, cat)\n            status = \"PASS\" if data and data.get(\"accepted\") else \"FAIL\"\n            print(f\"  [{status}] category/{slug}\")\n\n        for mid, m in list(self.dataset.merchants.items())[:5]:\n            data, err, _ = self.client.push_context(\"merchant\", mid, 1, m)\n            status = \"PASS\" if data and data.get(\"accepted\") else \"FAIL\"\n            short_id = mid.split('_')[1] if '_' in mid else mid[:10]\n            print(f\"  [{status}] merchant/{short_id}\")\n\n        return True\n\n    def _phase2_short(self) -> bool:\n        if not self._warmup():\n            return False\n\n        print_section(\"TICK TEST\")\n\n        trigs = list(self.dataset.triggers.keys())[:3]\n        for tid in trigs:\n            self.client.push_context(\"trigger\", tid, 1, self.dataset.triggers[tid])\n\n        data, err, lat = self.client.tick(trigs)\n        if err:\n            print_fail(f\"tick: {err}\")\n            return False\n\n        actions = data.get(\"actions\", [])\n        print_info(f\"Bot returned {len(actions)} action(s) ({lat:.0f}ms)\")\n\n        if not actions:\n            print_warn(\"No actions — bot chose not to send\")\n            return True\n\n        for action in actions:\n            self._score_and_display(action)\n\n        return True\n\n    def _auto_reply(self) -> bool:\n        print_section(\"AUTO-REPLY DETECTION\")\n\n        data, err, _ = self.client.healthz()\n        if err:\n            print_fail(f\"Bot unreachable: {err}\")\n            return False\n\n        mid = list(self.dataset.merchants.keys())[0] if self.dataset.merchants else \"m_test\"\n        auto_msg = \"Thank you for contacting us! Our team will respond shortly.\"\n\n        for i in range(1, 5):\n            print_info(f\"Turn {i}: Sending auto-reply...\")\n            data, err, _ = self.client.reply(f\"conv_auto_{i}\", mid, auto_msg, i + 1)\n\n            if err:\n                print_fail(f\"Error: {err}\")\n                return False\n\n            action = data.get(\"action\", \"?\")\n\n            if action == \"end\":\n                print_success(f\"Turn {i}: Bot ENDED — detected auto-reply pattern!\")\n                return True\n            elif action == \"wait\":\n                wait_s = data.get(\"wait_seconds\", \"?\")\n                print_success(f\"Turn {i}: Bot WAITING {wait_s}s\")\n            else:\n                body = data.get(\"body\", \"\")[:50]\n                print_warn(f\"Turn {i}: Bot sent: \\\"{body}...\\\"\")\n\n        print_warn(\"Bot never ended after 4 auto-replies\")\n        return True\n\n    def _intent(self) -> bool:\n        print_section(\"INTENT TRANSITION\")\n\n        data, err, _ = self.client.healthz()\n        if err:\n            print_fail(f\"Bot unreachable: {err}\")\n            return False\n\n        mid = list(self.dataset.merchants.keys())[0] if self.dataset.merchants else \"m_test\"\n        commitment = \"Ok lets do it. Whats next?\"\n\n        print_info(f\"Merchant: \\\"{commitment}\\\"\")\n        data, err, _ = self.client.reply(\"conv_intent_1\", mid, commitment, 2)\n\n        if err:\n            print_fail(f\"Error: {err}\")\n            return False\n\n        action = data.get(\"action\", \"?\")\n        body = data.get(\"body\", \"\")\n\n        print_info(f\"Bot action: {action}\")\n        if body:\n            print_info(f\"Bot body: \\\"{body[:100]}{'...' if len(body) > 100 else ''}\\\"\")\n\n        qualifying = [\"would you\", \"do you\", \"can you tell\", \"what if\", \"how about\"]\n        actioning = [\"done\", \"sending\", \"draft\", \"here\", \"confirm\", \"proceed\", \"next\"]\n\n        body_lower = body.lower()\n        if any(w in body_lower for w in actioning) and not any(w in body_lower for w in qualifying):\n            print_success(\"Bot correctly switched to ACTION mode\")\n        elif any(w in body_lower for w in qualifying):\n            print_fail(\"Bot is STILL QUALIFYING after commitment\")\n        else:\n            print_warn(\"Response unclear\")\n\n        return True\n\n    def _hostile(self) -> bool:\n        print_section(\"HOSTILE HANDLING\")\n\n        data, err, _ = self.client.healthz()\n        if err:\n            print_fail(f\"Bot unreachable: {err}\")\n            return False\n\n        mid = list(self.dataset.merchants.keys())[0] if self.dataset.merchants else \"m_test\"\n        hostile = \"Stop messaging me. This is useless spam.\"\n\n        print_info(f\"Merchant (hostile): \\\"{hostile}\\\"\")\n        data, err, _ = self.client.reply(\"conv_hostile\", mid, hostile, 2)\n\n        if err:\n            print_fail(f\"Error: {err}\")\n            return False\n\n        action = data.get(\"action\", \"?\")\n        body = data.get(\"body\", \"\")\n\n        print_info(f\"Bot action: {action}\")\n\n        if action == \"end\":\n            print_success(\"Bot correctly ENDED on hostile message\")\n        elif action == \"send\" and any(w in body.lower() for w in [\"sorry\", \"apolog\", \"won't\"]):\n            print_success(\"Bot apologized gracefully\")\n        else:\n            print_fail(\"Bot didn't handle hostility well\")\n\n        return True\n\n    def _all(self) -> bool:\n        results = []\n        for name, fn in [(\"warmup\", self._warmup), (\"auto_reply\", self._auto_reply),\n                         (\"intent\", self._intent), (\"hostile\", self._hostile)]:\n            try:\n                results.append((name, fn()))\n            except Exception as e:\n                print_fail(f\"{name} crashed: {e}\")\n                results.append((name, False))\n\n        print_section(\"SCENARIO RESULTS\")\n        for name, passed in results:\n            (print_success if passed else print_fail)(name)\n\n        return all(p for _, p in results)\n\n    def _full(self) -> bool:\n        if not self._warmup():\n            return False\n\n        print_section(\"FULL EVALUATION\")\n\n        for mid, m in self.dataset.merchants.items():\n            self.client.push_context(\"merchant\", mid, 1, m)\n        for tid, t in self.dataset.triggers.items():\n            self.client.push_context(\"trigger\", tid, 1, t)\n\n        print_success(\"All contexts pushed\")\n\n        print_section(\"SCORING COMPOSITIONS\")\n        tids = list(self.dataset.triggers.keys())\n\n        for i in range(0, len(tids), 5):\n            batch = tids[i:i+5]\n            data, err, lat = self.client.tick(batch)\n\n            if err:\n                print_warn(f\"Tick failed: {err}\")\n                continue\n\n            actions = data.get(\"actions\", [])\n            print_info(f\"Batch {i//5 + 1}: {len(actions)} actions ({lat:.0f}ms)\")\n\n            for action in actions:\n                self._score_and_display(action, verbose=False)\n\n        return True\n\n    def _score_and_display(self, action: Dict, verbose: bool = True):\n        \"\"\"Score an action and display results.\"\"\"\n        tid = action.get(\"trigger_id\", \"\")\n        mid = action.get(\"merchant_id\", \"\")\n        cid = action.get(\"customer_id\")\n\n        trigger = self.dataset.triggers.get(tid, {})\n        merchant = self.dataset.merchants.get(mid, {})\n        customer = self.dataset.customers.get(cid) if cid else None\n        category = self.dataset.categories.get(merchant.get(\"category_slug\", \"\"), {})\n\n        score = self.scorer.score(action, category, merchant, trigger, customer)\n        self.all_scores.append(score)\n\n        body = action.get(\"body\", \"\")[:50]\n        print(f\"\\n{Colors.CYAN}Message:{Colors.RESET} \\\"{body}...\\\"\")\n\n        print_score_bar(\"Specificity\", score.specificity)\n        if verbose and score.specificity_reason:\n            print_reason(score.specificity_reason)\n\n        print_score_bar(\"Category Fit\", score.category_fit)\n        if verbose and score.category_fit_reason:\n            print_reason(score.category_fit_reason)\n\n        print_score_bar(\"Merchant Fit\", score.merchant_fit)\n        if verbose and score.merchant_fit_reason:\n            print_reason(score.merchant_fit_reason)\n\n        print_score_bar(\"Decision Quality\", score.decision_quality)\n        if verbose and score.decision_quality_reason:\n            print_reason(score.decision_quality_reason)\n\n        print_score_bar(\"Engagement\", score.engagement_compulsion)\n        if verbose and score.engagement_reason:\n            print_reason(score.engagement_reason)\n\n        if score.penalties:\n            print(f\"  {Colors.RED}Penalties: -{score.penalties}{Colors.RESET}\")\n            for r in score.penalty_reasons:\n                print_reason(r)\n\n        print(f\"\\n  {Colors.BOLD}TOTAL: {score.total}/50{Colors.RESET}\")\n\n        if verbose and score.hint:\n            print_hint(score.hint)\n\n    def _final_summary(self):\n        if not self.all_scores:\n            return\n\n        print_section(\"FINAL SUMMARY\")\n\n        n = len(self.all_scores)\n        avg = ScoreResult(\n            specificity=sum(s.specificity for s in self.all_scores) // n,\n            category_fit=sum(s.category_fit for s in self.all_scores) // n,\n            merchant_fit=sum(s.merchant_fit for s in self.all_scores) // n,\n            decision_quality=sum(s.decision_quality for s in self.all_scores) // n,\n            engagement_compulsion=sum(s.engagement_compulsion for s in self.all_scores) // n,\n            penalties=sum(s.penalties for s in self.all_scores)\n        )\n\n        print_info(f\"Messages scored: {n}\\n\")\n\n        print_score_bar(\"Avg Specificity\", avg.specificity)\n        print_score_bar(\"Avg Category Fit\", avg.category_fit)\n        print_score_bar(\"Avg Merchant Fit\", avg.merchant_fit)\n        print_score_bar(\"Avg Decision Quality\", avg.decision_quality)\n        print_score_bar(\"Avg Engagement\", avg.engagement_compulsion)\n\n        total = avg.total\n        pct = (total / 50) * 100\n\n        print(f\"\\n{Colors.BOLD}  AVERAGE SCORE: {total}/50 ({pct:.0f}%){Colors.RESET}\")\n\n        if pct >= 80:\n            print(f\"\\n  {Colors.GREEN}EXCELLENT{Colors.RESET}\")\n        elif pct >= 60:\n            print(f\"\\n  {Colors.YELLOW}GOOD{Colors.RESET}\")\n        elif pct >= 40:\n            print(f\"\\n  {Colors.YELLOW}NEEDS IMPROVEMENT{Colors.RESET}\")\n        else:\n            print(f\"\\n  {Colors.RED}BELOW EXPECTATIONS{Colors.RESET}\")\n\n# =============================================================================\n# ENTRY POINT\n# =============================================================================\n\ndef main():\n    print_header(\"magicpin AI Challenge — LLM Judge\")\n    # Resolve API key and model from environment if not set in config\n    global LLM_API_KEY, LLM_MODEL, BOT_URL\n    if not LLM_API_KEY:\n        LLM_API_KEY = os.environ.get(\"OPENAI_API_KEY\", \"\")\n    if not LLM_MODEL:\n        LLM_MODEL = os.environ.get(\"OPENAI_MODEL\", \"\")\n    bot_port = os.environ.get(\"PORT\", \"\")\n    if bot_port and BOT_URL == \"http://localhost:8080\":\n        BOT_URL = f\"http://localhost:{bot_port}\"\n\n    # Validate configuration\n    if LLM_PROVIDER != \"ollama\" and not LLM_API_KEY:\n        print_fail(\"LLM_API_KEY is not set!\")\n        print_info(\"Edit the CONFIGURATION section at the top of this file\")\n        print_info(\"Set your API key for your chosen provider\")\n        sys.exit(1)\n\n    # Create LLM provider\n    try:\n        llm = create_provider()\n        print_info(f\"LLM Provider: {llm.name()}\")\n    except Exception as e:\n        print_fail(f\"Failed to create LLM provider: {e}\")\n        sys.exit(1)\n\n    # Test LLM connection\n    print_info(\"Testing LLM connection...\")\n    try:\n        test_response = llm.complete(\"Say 'ready' if you can hear me.\", \"You are a test assistant.\")\n        if test_response:\n            print_success(\"LLM connected successfully\")\n        else:\n            print_fail(\"LLM returned empty response\")\n            sys.exit(1)\n    except Exception as e:\n        print_fail(f\"LLM connection failed: {e}\")\n        print_info(\"Check your API key and internet connection\")\n        sys.exit(1)\n\n    # Run the judge\n    judge = JudgeSimulator(llm)\n    success = judge.run(TEST_SCENARIO)\n\n    sys.exit(0 if success else 1)\n\n\nif __name__ == \"__main__\":\n    main()\n"}, "fcfad6486dfe199e864bf3ffd594e30fa6a01e41": {"t": 1785763247.3577266, "n": 36150, "h": "a9911e1d38eabeab", "c": "#!/usr/bin/env python3\n\"\"\"\nmagicpin AI Challenge — LLM-Powered Judge Simulator\n====================================================\n\nA strict but fair judge that scores your bot and explains WHY.\n\nHOW TO USE:\n1. Edit the CONFIGURATION section below (lines 25-45)\n2. Set your LLM provider and API key\n3. Set your bot URL\n4. Run: python judge_simulator.py\n\nThat's it!\n\nAuthor: magicpin AI Challenge Team\n\"\"\"\n\n# =============================================================================\n# ██████  CONFIGURATION - EDIT THIS SECTION ██████\n# =============================================================================\n\n# Your bot's URL (where your bot is running)\nBOT_URL = \"http://localhost:8081\"\n\n# Choose your LLM provider: \"openai\", \"anthropic\", \"gemini\", \"deepseek\", \"groq\", \"ollama\", \"openrouter\"\nLLM_PROVIDER = \"openai\"\n\n# Your API key (paste your key here)\n\n# Model to use (leave empty for default, or specify like \"gpt-4o\", \"claude-3-5-sonnet-20241022\", etc.)\nLLM_MODEL = \"gpt-4.1-mini\"  # <-- Optional: specify model or leave empty for default\n\n# For Ollama only: local server URL\nOLLAMA_URL = \"http://localhost:11434\"\n\n# Which test to run by default\nTEST_SCENARIO = \"all\"\n\n# =============================================================================\n# ██████  END OF CONFIGURATION - DON'T EDIT BELOW THIS LINE ██████\n# =============================================================================\n\nimport os\nimport sys\nimport json\nimport time\nimport re\nimport socket\nfrom datetime import datetime\nfrom dataclasses import dataclass, field\nfrom typing import Optional, List, Dict, Any, Tuple\nfrom pathlib import Path\nfrom urllib import request as urlrequest, error as urlerror\nfrom abc import ABC, abstractmethod\n\n# Constants\nTIMEOUT_LLM = 45\nDATASET_DIR = Path(__file__).parent / \"dataset\"\n\n# =============================================================================\n# TERMINAL OUTPUT\n# =============================================================================\n\nclass Colors:\n    HEADER = '\\033[95m'\n    BLUE = '\\033[94m'\n    CYAN = '\\033[96m'\n    GREEN = '\\033[92m'\n    YELLOW = '\\033[93m'\n    RED = '\\033[91m'\n    MAGENTA = '\\033[35m'\n    BOLD = '\\033[1m'\n    DIM = '\\033[2m'\n    RESET = '\\033[0m'\n\ndef print_header(text: str):\n    print(f\"\\n{Colors.HEADER}{Colors.BOLD}{'='*70}{Colors.RESET}\")\n    print(f\"{Colors.HEADER}{Colors.BOLD}{text.center(70)}{Colors.RESET}\")\n    print(f\"{Colors.HEADER}{Colors.BOLD}{'='*70}{Colors.RESET}\\n\")\n\ndef print_section(text: str):\n    print(f\"\\n{Colors.CYAN}{Colors.BOLD}--- {text} ---{Colors.RESET}\\n\")\n\ndef print_success(text: str):\n    print(f\"{Colors.GREEN}[PASS]{Colors.RESET} {text}\")\n\ndef print_fail(text: str):\n    print(f\"{Colors.RED}[FAIL]{Colors.RESET} {text}\")\n\ndef print_warn(text: str):\n    print(f\"{Colors.YELLOW}[WARN]{Colors.RESET} {text}\")\n\ndef print_info(text: str):\n    print(f\"{Colors.BLUE}[INFO]{Colors.RESET} {text}\")\n\ndef print_llm(text: str):\n    print(f\"{Colors.MAGENTA}[LLM]{Colors.RESET} {text}\")\n\ndef print_score_bar(dimension: str, score: int, max_score: int = 10):\n    bar_filled = int((score / max_score) * 20)\n    bar_empty = 20 - bar_filled\n    color = Colors.GREEN if score >= 7 else Colors.YELLOW if score >= 4 else Colors.RED\n    print(f\"  {dimension:22} [{color}{'█' * bar_filled}{Colors.DIM}{'░' * bar_empty}{Colors.RESET}] {color}{score:2}/{max_score}{Colors.RESET}\")\n\ndef print_reason(text: str):\n    wrapped = text[:200] + \"...\" if len(text) > 200 else text\n    print(f\"    {Colors.DIM}{wrapped}{Colors.RESET}\")\n\ndef print_hint(hint: str):\n    print(f\"\\n  {Colors.YELLOW}Hint:{Colors.RESET} {hint}\")\n\n# =============================================================================\n# DATA CLASSES\n# =============================================================================\n\n@dataclass\nclass ScoreResult:\n    specificity: int = 0\n    specificity_reason: str = \"\"\n    category_fit: int = 0\n    category_fit_reason: str = \"\"\n    merchant_fit: int = 0\n    merchant_fit_reason: str = \"\"\n    decision_quality: int = 0\n    decision_quality_reason: str = \"\"\n    engagement_compulsion: int = 0\n    engagement_reason: str = \"\"\n    penalties: int = 0\n    penalty_reasons: List[str] = field(default_factory=list)\n    hint: str = \"\"\n\n    @property\n    def total(self) -> int:\n        return max(0, self.specificity + self.category_fit + self.merchant_fit +\n                   self.decision_quality + self.engagement_compulsion - self.penalties)\n\n# =============================================================================\n# LLM PROVIDERS\n# =============================================================================\n\nclass LLMProvider(ABC):\n    @abstractmethod\n    def complete(self, prompt: str, system: str = None) -> str:\n        pass\n\n    @abstractmethod\n    def name(self) -> str:\n        pass\n\n\nclass OpenAIProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"gpt-4o-mini\"\n\n    def name(self) -> str:\n        return f\"OpenAI ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        body = json.dumps({\n            \"model\": self.model,\n            \"messages\": messages,\n            \"temperature\": 0.2,\n            \"max_tokens\": 1500\n        }).encode(\"utf-8\")\n\n        req = urlrequest.Request(\n            \"https://api.openai.com/v1/chat/completions\",\n            data=body,\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\nclass AnthropicProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"claude-3-5-sonnet-20241022\"\n\n    def name(self) -> str:\n        return f\"Anthropic ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        body_dict = {\"model\": self.model, \"max_tokens\": 1500,\n                     \"messages\": [{\"role\": \"user\", \"content\": prompt}]}\n        if system:\n            body_dict[\"system\"] = system\n\n        req = urlrequest.Request(\n            \"https://api.anthropic.com/v1/messages\",\n            data=json.dumps(body_dict).encode(\"utf-8\"),\n            headers={\"x-api-key\": self.api_key, \"Content-Type\": \"application/json\",\n                     \"anthropic-version\": \"2023-06-01\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"content\"][0][\"text\"]\n\n\nclass GeminiProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"gemini-1.5-flash\"\n\n    def name(self) -> str:\n        return f\"Gemini ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        full_prompt = f\"{system}\\n\\n{prompt}\" if system else prompt\n        body = json.dumps({\n            \"contents\": [{\"parts\": [{\"text\": full_prompt}]}],\n            \"generationConfig\": {\"temperature\": 0.2, \"maxOutputTokens\": 1500}\n        }).encode(\"utf-8\")\n\n        url = f\"https://generativelanguage.googleapis.com/v1beta/models/{self.model}:generateContent?key={self.api_key}\"\n        req = urlrequest.Request(url, data=body, headers={\"Content-Type\": \"application/json\"})\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"candidates\"][0][\"content\"][\"parts\"][0][\"text\"]\n\n\nclass DeepSeekProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"deepseek-chat\"\n\n    def name(self) -> str:\n        return f\"DeepSeek ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        req = urlrequest.Request(\n            \"https://api.deepseek.com/v1/chat/completions\",\n            data=json.dumps({\"model\": self.model, \"messages\": messages,\n                            \"temperature\": 0.2, \"max_tokens\": 1500}).encode(\"utf-8\"),\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\nclass GroqProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"llama-3.1-70b-versatile\"\n\n    def name(self) -> str:\n        return f\"Groq ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        req = urlrequest.Request(\n            \"https://api.groq.com/openai/v1/chat/completions\",\n            data=json.dumps({\"model\": self.model, \"messages\": messages,\n                            \"temperature\": 0.2, \"max_tokens\": 1500}).encode(\"utf-8\"),\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\nclass OllamaProvider(LLMProvider):\n    def __init__(self, model: str = \"\", api_url: str = \"\"):\n        self.model = model or \"llama3\"\n        self.api_url = api_url or \"http://localhost:11434\"\n\n    def name(self) -> str:\n        return f\"Ollama ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        full_prompt = f\"{system}\\n\\n{prompt}\" if system else prompt\n        req = urlrequest.Request(\n            f\"{self.api_url}/api/generate\",\n            data=json.dumps({\"model\": self.model, \"prompt\": full_prompt,\n                            \"stream\": False, \"options\": {\"temperature\": 0.2}}).encode(\"utf-8\"),\n            headers={\"Content-Type\": \"application/json\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=90)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"response\"]\n\n\nclass OpenRouterProvider(LLMProvider):\n    def __init__(self, api_key: str, model: str = \"\"):\n        self.api_key = api_key\n        self.model = model or \"anthropic/claude-3-haiku\"\n\n    def name(self) -> str:\n        return f\"OpenRouter ({self.model})\"\n\n    def complete(self, prompt: str, system: str = None) -> str:\n        messages = []\n        if system:\n            messages.append({\"role\": \"system\", \"content\": system})\n        messages.append({\"role\": \"user\", \"content\": prompt})\n\n        req = urlrequest.Request(\n            \"https://openrouter.ai/api/v1/chat/completions\",\n            data=json.dumps({\"model\": self.model, \"messages\": messages,\n                            \"temperature\": 0.2, \"max_tokens\": 1500}).encode(\"utf-8\"),\n            headers={\"Authorization\": f\"Bearer {self.api_key}\", \"Content-Type\": \"application/json\",\n                     \"HTTP-Referer\": \"https://magicpin.com\"}\n        )\n        resp = urlrequest.urlopen(req, timeout=TIMEOUT_LLM)\n        data = json.loads(resp.read().decode(\"utf-8\"))\n        return data[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef create_provider() -> LLMProvider:\n    \"\"\"Create LLM provider from configuration.\"\"\"\n    providers = {\n        \"openai\": lambda: OpenAIProvider(LLM_API_KEY, LLM_MODEL),\n        \"anthropic\": lambda: AnthropicProvider(LLM_API_KEY, LLM_MODEL),\n        \"gemini\": lambda: GeminiProvider(LLM_API_KEY, LLM_MODEL),\n        \"deepseek\": lambda: DeepSeekProvider(LLM_API_KEY, LLM_MODEL),\n        \"groq\": lambda: GroqProvider(LLM_API_KEY, LLM_MODEL),\n        \"ollama\": lambda: OllamaProvider(LLM_MODEL, OLLAMA_URL),\n        \"openrouter\": lambda: OpenRouterProvider(LLM_API_KEY, LLM_MODEL),\n    }\n\n    if LLM_PROVIDER not in providers:\n        print_fail(f\"Unknown provider: {LLM_PROVIDER}\")\n        print_info(f\"Available: {', '.join(providers.keys())}\")\n        sys.exit(1)\n\n    return providers[LLM_PROVIDER]()\n\n# =============================================================================\n# DATASET & BOT CLIENT\n# =============================================================================\n\nclass DatasetLoader:\n    def __init__(self, dataset_dir: Path):\n        self.dataset_dir = dataset_dir\n        self.categories = {}\n        self.merchants = {}\n        self.customers = {}\n        self.triggers = {}\n\n    def load(self) -> bool:\n        try:\n            cat_dir = self.dataset_dir / \"categories\"\n            if cat_dir.exists():\n                for f in cat_dir.glob(\"*.json\"):\n                    data = json.load(open(f))\n                    self.categories[data.get(\"slug\", f.stem)] = data\n\n            for name, container, key in [\n                (\"merchants_seed.json\", \"merchants\", \"merchant_id\"),\n                (\"customers_seed.json\", \"customers\", \"customer_id\"),\n                (\"triggers_seed.json\", \"triggers\", \"id\")\n            ]:\n                path = self.dataset_dir / name\n                if path.exists():\n                    data = json.load(open(path))\n                    items = data.get(container, data.get(container.rstrip(\"s\"), []))\n                    storage = getattr(self, container)\n                    for item in items:\n                        if key in item:\n                            storage[item[key]] = item\n            return True\n        except Exception as e:\n            print_fail(f\"Dataset load error: {e}\")\n            return False\n\n\nclass BotClient:\n    def __init__(self, base_url: str):\n        self.base_url = base_url.rstrip(\"/\")\n\n    def _request(self, method: str, path: str, timeout: int = 30,\n                 body_dict: Dict = None) -> Tuple[Optional[Dict], Optional[str], float]:\n        url = f\"{self.base_url}{path}\"\n        start = time.time()\n        body = json.dumps(body_dict).encode(\"utf-8\") if body_dict else None\n        headers = {\"Content-Type\": \"application/json\"}\n        req = urlrequest.Request(url, data=body, method=method, headers=headers)\n\n        try:\n            resp = urlrequest.urlopen(req, timeout=timeout)\n            return json.loads(resp.read().decode(\"utf-8\")), None, (time.time() - start) * 1000\n        except urlerror.HTTPError as e:\n            latency = (time.time() - start) * 1000\n            if e.code == 401:\n                return None, \"Unauthorized\", latency\n            try:\n                return json.loads(e.read().decode(\"utf-8\")), None, latency\n            except:\n                return None, f\"HTTP {e.code}\", latency\n        except Exception as e:\n            return None, str(e), (time.time() - start) * 1000\n\n    def healthz(self):\n        return self._request(\"GET\", \"/v1/healthz\", 5)\n\n    def metadata(self):\n        return self._request(\"GET\", \"/v1/metadata\", 5)\n\n    def push_context(self, scope, cid, version, payload):\n        return self._request(\"POST\", \"/v1/context\", 10, {\n            \"scope\": scope, \"context_id\": cid, \"version\": version,\n            \"payload\": payload, \"delivered_at\": datetime.utcnow().isoformat() + \"Z\"\n        })\n\n    def tick(self, triggers):\n        return self._request(\"POST\", \"/v1/tick\", 15, {\n            \"now\": datetime.utcnow().isoformat() + \"Z\", \"available_triggers\": triggers\n        })\n\n    def reply(self, conv_id, merchant_id, message, turn):\n        return self._request(\"POST\", \"/v1/reply\", 15, {\n            \"conversation_id\": conv_id, \"merchant_id\": merchant_id, \"customer_id\": None,\n            \"from_role\": \"merchant\", \"message\": message,\n            \"received_at\": datetime.utcnow().isoformat() + \"Z\", \"turn_number\": turn\n        })\n\n# =============================================================================\n# LLM SCORING ENGINE\n# =============================================================================\n\nclass LLMScorer:\n    \"\"\"Scores messages using LLM and provides detailed reasoning.\"\"\"\n\n    SYSTEM = \"\"\"You are a STRICT judge for the magicpin AI Challenge. You score merchant engagement messages.\n\nSCORING DIMENSIONS (0-10 each, be strict - 5 is average, 7+ is good, 9+ is excellent):\n\n1. SPECIFICITY: Does the message have VERIFIABLE facts?\n   - Numbers (percentages, counts, prices)\n   - Dates/times\n   - Source citations\n   - Concrete claims vs vague statements\n\n2. CATEGORY FIT: Does the voice match the business type?\n   - Dentists: clinical, peer-to-peer, technical OK, use \"Dr.\" prefix\n   - Salons: warm, friendly, practical\n   - Restaurants: operator-to-operator\n   - Gyms: coaching, motivational\n   - Pharmacies: trustworthy, precise\n\n3. MERCHANT FIT: Is it personalized to THIS merchant?\n   - Uses their name/owner name correctly\n   - References their actual data (not fabricated)\n   - Honors language preference\n\n4. TRIGGER RELEVANCE: Does it connect to WHY NOW?\n   - Clear reason for this specific message\n   - Uses data from the trigger payload\n   - Not a generic nudge\n\n5. ENGAGEMENT COMPULSION: Would they reply?\n   - Loss aversion, curiosity, social proof\n   - Clear CTA\n   - Low friction ask\n\nPENALTIES:\n- Fabricating data not in context: -2\n- Exposing internal jargon to merchant: -1\n\nRESPOND ONLY WITH THIS EXACT JSON FORMAT:\n{\n  \"specificity\": <0-10>,\n  \"specificity_reason\": \"<why this score, 1-2 sentences>\",\n  \"category_fit\": <0-10>,\n  \"category_fit_reason\": \"<why this score>\",\n  \"merchant_fit\": <0-10>,\n  \"merchant_fit_reason\": \"<why this score>\",\n  \"decision_quality\": <0-10>,\n  \"decision_quality_reason\": \"<why this score>\",\n  \"engagement_compulsion\": <0-10>,\n  \"engagement_reason\": \"<why this score>\",\n  \"hint\": \"<one sentence guidance for improvement, cryptic not direct>\"\n}\"\"\"\n\n    def __init__(self, llm: LLMProvider, dataset: DatasetLoader):\n        self.llm = llm\n        self.dataset = dataset\n\n    def score(self, action: Dict, category: Dict, merchant: Dict,\n              trigger: Dict, customer: Dict = None) -> ScoreResult:\n        \"\"\"Score a message and return detailed results.\"\"\"\n\n        body = action.get(\"body\", \"\")\n\n        prompt = f\"\"\"SCORE THIS MESSAGE:\n\n=== CONTEXT PROVIDED TO BOT ===\nCategory: {category.get('slug', 'unknown')}\nVoice: {category.get('voice', {}).get('tone', 'unknown')}\nTaboos: {category.get('voice', {}).get('vocab_taboo', [])[:5]}\n\nMerchant: {merchant.get('identity', {}).get('name', 'unknown')}\nOwner: {merchant.get('identity', {}).get('owner_first_name', 'unknown')}\nLocality: {merchant.get('identity', {}).get('locality', 'unknown')}\nLanguages: {merchant.get('identity', {}).get('languages', [])}\nPerformance: views={merchant.get('performance', {}).get('views', '?')}, calls={merchant.get('performance', {}).get('calls', '?')}, ctr={merchant.get('performance', {}).get('ctr', '?')}\nSignals: {merchant.get('signals', [])}\nActive Offers: {[o.get('title') for o in merchant.get('offers', []) if o.get('status') == 'active']}\n\nTrigger Kind: {trigger.get('kind', 'unknown')}\nTrigger Payload: {json.dumps(trigger.get('payload', {}))}\nTrigger Urgency: {trigger.get('urgency', '?')}\n\nCustomer: {json.dumps(customer.get('identity', {})) if customer else 'None (merchant-facing)'}\n\n=== BOT'S MESSAGE ===\nBody ({len(body)} chars): \"{body}\"\nCTA: {action.get('cta', 'none')}\nSend As: {action.get('send_as', 'vera')}\n\nScore each dimension 0-10 with clear reasoning. Be STRICT.\"\"\"\n\n        try:\n            print_llm(\"Analyzing message...\")\n            response = self.llm.complete(prompt, self.SYSTEM)\n            return self._parse_response(response, action)\n        except Exception as e:\n            print_warn(f\"LLM error: {e}\")\n            return self._fallback_score(action)\n\n    def _parse_response(self, response: str, action: Dict) -> ScoreResult:\n        \"\"\"Parse LLM JSON response.\"\"\"\n        match = re.search(r'\\{[\\s\\S]*\\}', response)\n        if not match:\n            return self._fallback_score(action)\n\n        try:\n            data = json.loads(match.group())\n            result = ScoreResult(\n                specificity=min(10, max(0, int(data.get(\"specificity\", 5)))),\n                specificity_reason=data.get(\"specificity_reason\", \"\"),\n                category_fit=min(10, max(0, int(data.get(\"category_fit\", 5)))),\n                category_fit_reason=data.get(\"category_fit_reason\", \"\"),\n                merchant_fit=min(10, max(0, int(data.get(\"merchant_fit\", 5)))),\n                merchant_fit_reason=data.get(\"merchant_fit_reason\", \"\"),\n                decision_quality=min(10, max(0, int(data.get(\"decision_quality\", data.get(\"trigger_relevance\", 5))))),\n                decision_quality_reason=data.get(\"decision_quality_reason\", data.get(\"trigger_relevance_reason\", \"\")),\n                engagement_compulsion=min(10, max(0, int(data.get(\"engagement_compulsion\", 5)))),\n                engagement_reason=data.get(\"engagement_reason\", \"\"),\n                hint=data.get(\"hint\", \"\")\n            )\n            return result\n        except Exception as e:\n            print_warn(f\"Parse error: {e}\")\n            return self._fallback_score(action)\n\n    def _fallback_score(self, action: Dict) -> ScoreResult:\n        \"\"\"Basic fallback scoring.\"\"\"\n        body = action.get(\"body\", \"\").lower()\n        nums = len(re.findall(r'\\d+', body))\n        return ScoreResult(\n            specificity=min(10, 3 + nums * 2),\n            specificity_reason=\"Fallback: counted numbers in message\",\n            category_fit=5, category_fit_reason=\"Could not evaluate\",\n            merchant_fit=5, merchant_fit_reason=\"Could not evaluate\",\n            decision_quality=5, decision_quality_reason=\"Could not evaluate\",\n            engagement_compulsion=5, engagement_reason=\"Could not evaluate\",\n            hint=\"LLM scoring failed - using basic heuristics\"\n        )\n\n# =============================================================================\n# MAIN JUDGE\n# =============================================================================\n\nclass JudgeSimulator:\n    def __init__(self, llm: LLMProvider):\n        self.llm = llm\n        self.client = BotClient(BOT_URL)\n        self.dataset = DatasetLoader(DATASET_DIR)\n        self.scorer: Optional[LLMScorer] = None\n        self.all_scores: List[ScoreResult] = []\n\n    def run(self, scenario: str) -> bool:\n        print_header(f\"LLM JUDGE — {scenario.upper()}\")\n        print_info(f\"Bot: {BOT_URL}\")\n        print_info(f\"LLM: {self.llm.name()}\")\n\n        if not self.dataset.load():\n            print_fail(\"Dataset load failed\")\n            return False\n\n        self.scorer = LLMScorer(self.llm, self.dataset)\n        print_info(f\"Loaded: {len(self.dataset.categories)} categories, \"\n                   f\"{len(self.dataset.merchants)} merchants, \"\n                   f\"{len(self.dataset.triggers)} triggers\")\n\n        scenarios = {\n            \"warmup\": self._warmup,\n            \"phase2_short\": self._phase2_short,\n            \"auto_reply_hell\": self._auto_reply,\n            \"intent_transition\": self._intent,\n            \"hostile\": self._hostile,\n            \"all\": self._all,\n            \"full_evaluation\": self._full,\n        }\n\n        if scenario not in scenarios:\n            print_fail(f\"Unknown scenario: {scenario}\")\n            print_info(f\"Available: {', '.join(scenarios.keys())}\")\n            return False\n\n        success = scenarios[scenario]()\n        self._final_summary()\n        return success\n\n    def _warmup(self) -> bool:\n        print_section(\"WARMUP\")\n\n        data, err, lat = self.client.healthz()\n        if err:\n            print_fail(f\"healthz: {err}\")\n            return False\n        print_success(f\"healthz ({lat:.0f}ms)\")\n\n        data, err, lat = self.client.metadata()\n        if err:\n            print_warn(f\"metadata: {err}\")\n        else:\n            print_success(f\"metadata — Team: {data.get('team_name', '?')}, Model: {data.get('model', '?')}\")\n\n        print_section(\"CONTEXT PUSH\")\n        for slug, cat in self.dataset.categories.items():\n            data, err, _ = self.client.push_context(\"category\", slug, 1, cat)\n            status = \"PASS\" if data and data.get(\"accepted\") else \"FAIL\"\n            print(f\"  [{status}] category/{slug}\")\n\n        for mid, m in list(self.dataset.merchants.items())[:5]:\n            data, err, _ = self.client.push_context(\"merchant\", mid, 1, m)\n            status = \"PASS\" if data and data.get(\"accepted\") else \"FAIL\"\n            short_id = mid.split('_')[1] if '_' in mid else mid[:10]\n            print(f\"  [{status}] merchant/{short_id}\")\n\n        return True\n\n    def _phase2_short(self) -> bool:\n        if not self._warmup():\n            return False\n\n        print_section(\"TICK TEST\")\n\n        trigs = list(self.dataset.triggers.keys())[:3]\n        for tid in trigs:\n            self.client.push_context(\"trigger\", tid, 1, self.dataset.triggers[tid])\n\n        data, err, lat = self.client.tick(trigs)\n        if err:\n            print_fail(f\"tick: {err}\")\n            return False\n\n        actions = data.get(\"actions\", [])\n        print_info(f\"Bot returned {len(actions)} action(s) ({lat:.0f}ms)\")\n\n        if not actions:\n            print_warn(\"No actions — bot chose not to send\")\n            return True\n\n        for action in actions:\n            self._score_and_display(action)\n\n        return True\n\n    def _auto_reply(self) -> bool:\n        print_section(\"AUTO-REPLY DETECTION\")\n\n        data, err, _ = self.client.healthz()\n        if err:\n            print_fail(f\"Bot unreachable: {err}\")\n            return False\n\n        mid = list(self.dataset.merchants.keys())[0] if self.dataset.merchants else \"m_test\"\n        auto_msg = \"Thank you for contacting us! Our team will respond shortly.\"\n\n        for i in range(1, 5):\n            print_info(f\"Turn {i}: Sending auto-reply...\")\n            data, err, _ = self.client.reply(f\"conv_auto_{i}\", mid, auto_msg, i + 1)\n\n            if err:\n                print_fail(f\"Error: {err}\")\n                return False\n\n            action = data.get(\"action\", \"?\")\n\n            if action == \"end\":\n                print_success(f\"Turn {i}: Bot ENDED — detected auto-reply pattern!\")\n                return True\n            elif action == \"wait\":\n                wait_s = data.get(\"wait_seconds\", \"?\")\n                print_success(f\"Turn {i}: Bot WAITING {wait_s}s\")\n            else:\n                body = data.get(\"body\", \"\")[:50]\n                print_warn(f\"Turn {i}: Bot sent: \\\"{body}...\\\"\")\n\n        print_warn(\"Bot never ended after 4 auto-replies\")\n        return True\n\n    def _intent(self) -> bool:\n        print_section(\"INTENT TRANSITION\")\n\n        data, err, _ = self.client.healthz()\n        if err:\n            print_fail(f\"Bot unreachable: {err}\")\n            return False\n\n        mid = list(self.dataset.merchants.keys())[0] if self.dataset.merchants else \"m_test\"\n        commitment = \"Ok lets do it. Whats next?\"\n\n        print_info(f\"Merchant: \\\"{commitment}\\\"\")\n        data, err, _ = self.client.reply(\"conv_intent_1\", mid, commitment, 2)\n\n        if err:\n            print_fail(f\"Error: {err}\")\n            return False\n\n        action = data.get(\"action\", \"?\")\n        body = data.get(\"body\", \"\")\n\n        print_info(f\"Bot action: {action}\")\n        if body:\n            print_info(f\"Bot body: \\\"{body[:100]}{'...' if len(body) > 100 else ''}\\\"\")\n\n        qualifying = [\"would you\", \"do you\", \"can you tell\", \"what if\", \"how about\"]\n        actioning = [\"done\", \"sending\", \"draft\", \"here\", \"confirm\", \"proceed\", \"next\"]\n\n        body_lower = body.lower()\n        if any(w in body_lower for w in actioning) and not any(w in body_lower for w in qualifying):\n            print_success(\"Bot correctly switched to ACTION mode\")\n        elif any(w in body_lower for w in qualifying):\n            print_fail(\"Bot is STILL QUALIFYING after commitment\")\n        else:\n            print_warn(\"Response unclear\")\n\n        return True\n\n    def _hostile(self) -> bool:\n        print_section(\"HOSTILE HANDLING\")\n\n        data, err, _ = self.client.healthz()\n        if err:\n            print_fail(f\"Bot unreachable: {err}\")\n            return False\n\n        mid = list(self.dataset.merchants.keys())[0] if self.dataset.merchants else \"m_test\"\n        hostile = \"Stop messaging me. This is useless spam.\"\n\n        print_info(f\"Merchant (hostile): \\\"{hostile}\\\"\")\n        data, err, _ = self.client.reply(\"conv_hostile\", mid, hostile, 2)\n\n        if err:\n            print_fail(f\"Error: {err}\")\n            return False\n\n        action = data.get(\"action\", \"?\")\n        body = data.get(\"body\", \"\")\n\n        print_info(f\"Bot action: {action}\")\n\n        if action == \"end\":\n            print_success(\"Bot correctly ENDED on hostile message\")\n        elif action == \"send\" and any(w in body.lower() for w in [\"sorry\", \"apolog\", \"won't\"]):\n            print_success(\"Bot apologized gracefully\")\n        else:\n            print_fail(\"Bot didn't handle hostility well\")\n\n        return True\n\n    def _all(self) -> bool:\n        results = []\n        for name, fn in [(\"warmup\", self._warmup), (\"auto_reply\", self._auto_reply),\n                         (\"intent\", self._intent), (\"hostile\", self._hostile)]:\n            try:\n                results.append((name, fn()))\n            except Exception as e:\n                print_fail(f\"{name} crashed: {e}\")\n                results.append((name, False))\n\n        print_section(\"SCENARIO RESULTS\")\n        for name, passed in results:\n            (print_success if passed else print_fail)(name)\n\n        return all(p for _, p in results)\n\n    def _full(self) -> bool:\n        if not self._warmup():\n            return False\n\n        print_section(\"FULL EVALUATION\")\n\n        for mid, m in self.dataset.merchants.items():\n            self.client.push_context(\"merchant\", mid, 1, m)\n        for tid, t in self.dataset.triggers.items():\n            self.client.push_context(\"trigger\", tid, 1, t)\n\n        print_success(\"All contexts pushed\")\n\n        print_section(\"SCORING COMPOSITIONS\")\n        tids = list(self.dataset.triggers.keys())\n\n        for i in range(0, len(tids), 5):\n            batch = tids[i:i+5]\n            data, err, lat = self.client.tick(batch)\n\n            if err:\n                print_warn(f\"Tick failed: {err}\")\n                continue\n\n            actions = data.get(\"actions\", [])\n            print_info(f\"Batch {i//5 + 1}: {len(actions)} actions ({lat:.0f}ms)\")\n\n            for action in actions:\n                self._score_and_display(action, verbose=False)\n\n        return True\n\n    def _score_and_display(self, action: Dict, verbose: bool = True):\n        \"\"\"Score an action and display results.\"\"\"\n        tid = action.get(\"trigger_id\", \"\")\n        mid = action.get(\"merchant_id\", \"\")\n        cid = action.get(\"customer_id\")\n\n        trigger = self.dataset.triggers.get(tid, {})\n        merchant = self.dataset.merchants.get(mid, {})\n        customer = self.dataset.customers.get(cid) if cid else None\n        category = self.dataset.categories.get(merchant.get(\"category_slug\", \"\"), {})\n\n        score = self.scorer.score(action, category, merchant, trigger, customer)\n        self.all_scores.append(score)\n\n        body = action.get(\"body\", \"\")[:50]\n        print(f\"\\n{Colors.CYAN}Message:{Colors.RESET} \\\"{body}...\\\"\")\n\n        print_score_bar(\"Specificity\", score.specificity)\n        if verbose and score.specificity_reason:\n            print_reason(score.specificity_reason)\n\n        print_score_bar(\"Category Fit\", score.category_fit)\n        if verbose and score.category_fit_reason:\n            print_reason(score.category_fit_reason)\n\n        print_score_bar(\"Merchant Fit\", score.merchant_fit)\n        if verbose and score.merchant_fit_reason:\n            print_reason(score.merchant_fit_reason)\n\n        print_score_bar(\"Decision Quality\", score.decision_quality)\n        if verbose and score.decision_quality_reason:\n            print_reason(score.decision_quality_reason)\n\n        print_score_bar(\"Engagement\", score.engagement_compulsion)\n        if verbose and score.engagement_reason:\n            print_reason(score.engagement_reason)\n\n        if score.penalties:\n            print(f\"  {Colors.RED}Penalties: -{score.penalties}{Colors.RESET}\")\n            for r in score.penalty_reasons:\n                print_reason(r)\n\n        print(f\"\\n  {Colors.BOLD}TOTAL: {score.total}/50{Colors.RESET}\")\n\n        if verbose and score.hint:\n            print_hint(score.hint)\n\n    def _final_summary(self):\n        if not self.all_scores:\n            return\n\n        print_section(\"FINAL SUMMARY\")\n\n        n = len(self.all_scores)\n        avg = ScoreResult(\n            specificity=sum(s.specificity for s in self.all_scores) // n,\n            category_fit=sum(s.category_fit for s in self.all_scores) // n,\n            merchant_fit=sum(s.merchant_fit for s in self.all_scores) // n,\n            decision_quality=sum(s.decision_quality for s in self.all_scores) // n,\n            engagement_compulsion=sum(s.engagement_compulsion for s in self.all_scores) // n,\n            penalties=sum(s.penalties for s in self.all_scores)\n        )\n\n        print_info(f\"Messages scored: {n}\\n\")\n\n        print_score_bar(\"Avg Specificity\", avg.specificity)\n        print_score_bar(\"Avg Category Fit\", avg.category_fit)\n        print_score_bar(\"Avg Merchant Fit\", avg.merchant_fit)\n        print_score_bar(\"Avg Decision Quality\", avg.decision_quality)\n        print_score_bar(\"Avg Engagement\", avg.engagement_compulsion)\n\n        total = avg.total\n        pct = (total / 50) * 100\n\n        print(f\"\\n{Colors.BOLD}  AVERAGE SCORE: {total}/50 ({pct:.0f}%){Colors.RESET}\")\n\n        if pct >= 80:\n            print(f\"\\n  {Colors.GREEN}EXCELLENT{Colors.RESET}\")\n        elif pct >= 60:\n            print(f\"\\n  {Colors.YELLOW}GOOD{Colors.RESET}\")\n        elif pct >= 40:\n            print(f\"\\n  {Colors.YELLOW}NEEDS IMPROVEMENT{Colors.RESET}\")\n        else:\n            print(f\"\\n  {Colors.RED}BELOW EXPECTATIONS{Colors.RESET}\")\n\n# =============================================================================\n# ENTRY POINT\n# =============================================================================\n\ndef main():\n    print_header(\"magicpin AI Challenge — LLM Judge\")\n    # Resolve API key and model from environment if not set in config\n    global LLM_API_KEY, LLM_MODEL, BOT_URL\n    if not LLM_API_KEY:\n        LLM_API_KEY = os.environ.get(\"OPENAI_API_KEY\", \"\")\n    if not LLM_MODEL:\n        LLM_MODEL = os.environ.get(\"OPENAI_MODEL\", \"\")\n    bot_port = os.environ.get(\"PORT\", \"\")\n    if bot_port and BOT_URL == \"http://localhost:8080\":\n        BOT_URL = f\"http://localhost:{bot_port}\"\n\n    # Validate configuration\n    if LLM_PROVIDER != \"ollama\" and not LLM_API_KEY:\n        print_fail(\"LLM_API_KEY is not set!\")\n        print_info(\"Edit the CONFIGURATION section at the top of this file\")\n        print_info(\"Set your API key for your chosen provider\")\n        sys.exit(1)\n\n    # Create LLM provider\n    try:\n        llm = create_provider()\n        print_info(f\"LLM Provider: {llm.name()}\")\n    except Exception as e:\n        print_fail(f\"Failed to create LLM provider: {e}\")\n        sys.exit(1)\n\n    # Test LLM connection\n    print_info(\"Testing LLM connection...\")\n    try:\n        test_response = llm.complete(\"Say 'ready' if you can hear me.\", \"You are a test assistant.\")\n        if test_response:\n            print_success(\"LLM connected successfully\")\n        else:\n            print_fail(\"LLM returned empty response\")\n            sys.exit(1)\n    except Exception as e:\n        print_fail(f\"LLM connection failed: {e}\")\n        print_info(\"Check your API key and internet connection\")\n        sys.exit(1)\n\n    # Run the judge\n    judge = JudgeSimulator(llm)\n    success = judge.run(TEST_SCENARIO)\n\n    sys.exit(0 if success else 1)\n\n\nif __name__ == \"__main__\":\n    main()\n"}}}}