{"repo": "battatawada/cdc-automate", "files": {"vision-for-opencode/vision_proxy.py": {"f8932eb2e84c6efa190533cbda133335b287324f": {"t": 1785674495.9298184, "n": 83871, "h": "d7356df58fe85708", "c": "#!/usr/bin/env python3\n# Vision Tool — Image & video analysis for AI coding assistants\n# Copyright (c) 2026 Farhan Dhrubo \n# License: GPL-3.0 — https://github.com/farhanic017/vision-tool\n#\n# This program is free software. You may NOT remove this notice,\n# re-distribute as your own work, or sell without attribution.\n# =============================================================================\n\n\"\"\"\nvision_proxy.py — Image & video analysis for AI models without native vision.\nCopyright (C) 2026 Farhan Dhrubo\n\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nHandles:\n - Images (png, jpg, webp, bmp, gif)\n - Videos (mp4, webm, mov, avi, mkv, flv, wmv, m4v) via ffmpeg keyframe extraction\n\nTries free backends first, then falls back to others in parallel.\n Priority: Gemini → Azure → Groq → HuggingFace → Mistral in parallel\n Total timeout: 25s | Per-backend timeout: 12s\n\nCustom model (auto-routes to best provider):\n --model \"azureai/Phi-4-multimodal-instruct\" → Azure AI Foundry\n --model \"cf/@cf/google/gemma-4-26b-a4b-it\" → Cloudflare Workers AI\n --model \"groq/meta-llama/llama-4-scout-17b-16e-instruct\" → Groq\n Set VISION_MODEL env var or DEFAULT_MODEL in config.json for persistence.\n\nSupported provider keys (set via setup.py or env vars):\n GEMINI_API_KEY | CLOUDFLARE_API_KEY | AZUREAI_API_KEY | AZUREAI_ENDPOINT | ANTHROPIC_API_KEY | MISTRAL_API_KEY | GROQ_API_KEY | HF_TOKEN | FIREWORKS_API_KEY | ZAI_API_KEY\n\nUsage:\n python vision_proxy.py [prompt text...] [--model NAME]\n\nFirst run? Run setup.py to configure your API keys:\n python setup.py\n\"\"\"\n\n\nimport base64\nimport json\nimport os\nimport sys\nimport io\nimport mimetypes\nimport urllib.request\nimport urllib.error\nimport subprocess\nimport tempfile\nimport shutil\nimport string\nimport time\nimport concurrent.futures\nimport threading\n\n# ── UTF-8 stdout wrapper (Windows cp1252 fix) — module level ──────────\n\ndef _wrap_stream_utf8(stream):\n if sys.platform != \"win32\" or not hasattr(stream, \"buffer\"):\n return stream\n encoding = (getattr(stream, \"encoding\", \"\") or \"\").lower().replace(\"-\", \"\")\n if encoding == \"utf8\":\n return stream\n try:\n buffer = stream.detach()\n return io.TextIOWrapper(buffer, encoding=\"utf-8\", errors=\"replace\", line_buffering=True)\n except (AttributeError, TypeError, ValueError, OSError):\n return stream\n\n\nsys.stdout = _wrap_stream_utf8(sys.stdout)\n\n\ndef _wrap_utf8():\n \"\"\"Idempotently wrap stdout as UTF-8 on Windows.\"\"\"\n sys.stdout = _wrap_stream_utf8(sys.stdout)\n\n\n# ── File search — fast, simple, user-dirs only ───────────────────────────\n_SEARCH_CACHE = {}\n_GLOBAL_SEARCH_TIMEOUT = 5 # seconds for optional recursive fallback\n\n\ndef _get_search_dirs():\n \"\"\"Known user directories to check for files. No drive scanning.\"\"\"\n key = \"search_dirs\"\n if key in _SEARCH_CACHE:\n return _SEARCH_CACHE[key]\n dirs = set()\n username = os.environ.get(\"USERNAME\", \"\")\n for drive_letter in string.ascii_uppercase:\n drive = f\"{drive_letter}:\"\n if not os.path.isdir(drive):\n continue\n if username:\n for sub in (\"Desktop\", \"Downloads\", \"Pictures\", \"Documents\",\n \"Pictures\\\\Screenshots\", \"AppData\\\\Roaming\\\\Microsoft\\\\Windows\\\\Recent\"):\n p = os.path.join(drive, \"Users\", username, sub)\n if os.path.isdir(p):\n dirs.add(os.path.abspath(p))\n try:\n dirs.add(os.path.abspath(os.getcwd()))\n except Exception:\n pass\n home = os.path.abspath(os.path.expanduser(\"~\"))\n if os.path.isdir(home):\n dirs.add(home)\n result = sorted(dirs)\n _SEARCH_CACHE[key] = result\n return result\n\n\ndef _should_skip_dir(name):\n lower = name.lower()\n if lower in {\"$recycle.bin\", \"windows\", \"winnt\", \"program files\",\n \"program files (x86)\", \"programdata\", \"boot\", \"recovery\",\n \"perflogs\", \"system volume information\"}:\n return True\n return name.startswith(\"$\") or name.startswith(\".\")\n\n\ndef _scandir_walk(root_dir, filename, deadline, max_depth=3):\n \"\"\"Quick recursive file search with deadline. Depth-limited.\"\"\"\n if not os.path.isdir(root_dir):\n return\n root_dir = os.path.abspath(root_dir)\n file_lower = filename.lower()\n queue = [(root_dir, 0)]\n while queue and time.time() < deadline:\n dirpath, depth = queue.pop(0)\n if depth > max_depth:\n continue\n try:\n with os.scandir(dirpath) as entries:\n for entry in entries:\n if time.time() >= deadline:\n return\n try:\n is_dir = entry.is_dir(follow_symlinks=False)\n except OSError:\n continue\n if is_dir:\n if depth < max_depth and not _should_skip_dir(entry.name):\n queue.append((entry.path, depth + 1))\n elif entry.name.lower() == file_lower:\n yield os.path.abspath(entry.path)\n except (PermissionError, OSError):\n continue\n\n\ndef find_file(name, max_results=5):\n \"\"\"Fast file search — checks direct path, then user dirs, then shallow recursive.\"\"\"\n if not name:\n return []\n name = name.strip().strip('\"\\'').strip()\n basename = os.path.basename(name)\n if not basename:\n return []\n\n abs_check = os.path.abspath(name)\n if os.path.isfile(abs_check):\n return [abs_check]\n\n cache_key = (\"find\", basename)\n if cache_key in _SEARCH_CACHE:\n return _SEARCH_CACHE[cache_key]\n\n dirs = _get_search_dirs()\n for d in dirs:\n candidate = os.path.join(d, basename)\n if os.path.isfile(candidate):\n result = [os.path.abspath(candidate)]\n _SEARCH_CACHE[cache_key] = result\n return result\n\n deadline = time.time() + _GLOBAL_SEARCH_TIMEOUT\n for d in dirs:\n if time.time() >= deadline:\n break\n for match in _scandir_walk(d, basename, deadline, max_depth=3):\n _SEARCH_CACHE[cache_key] = [match]\n return [match]\n\n _SEARCH_CACHE[cache_key] = []\n return []\n\n\n# ── Config loader ────────────────────────────────────────────────────────\n_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))\n_APPDATA_DIR = os.path.join(os.environ.get(\"APPDATA\", os.path.expanduser(\"~\")), \"vision-tool\")\nCONFIG_PATH = os.path.join(_APPDATA_DIR, \"config.json\")\nCONFIG_PATH_LOCAL = os.path.join(_SCRIPT_DIR, \"config.json\")\nBACKEND_MEMORY_PATH = os.path.join(_APPDATA_DIR, \"backend_memory.json\")\nBACKEND_LIMIT_COOLDOWN = 24 * 60 * 60\nCAPABILITY_REFRESH_INTERVAL = 2 * 24 * 60 * 60\nCAPABILITY_REFRESH_STALE_AFTER = 30 * 60\nCAPABILITY_PROBE_TIMEOUT = 6\nTINY_PNG_B64 = (\n \"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4DwAAAQEABRjYTgAAAABJRU5ErkJggg==\"\n)\n\n\nALL_PROVIDER_KEYS = [\n \"CLOUDFLARE_API_KEY\", \"AZUREAI_API_KEY\", \"AZUREAI_ENDPOINT\",\n \"GROQ_API_KEY\", \"HF_TOKEN\", \"MISTRAL_API_KEY\", \"GEMINI_API_KEY\",\n \"OPENROUTER_API_KEY\", \"OPENAI_API_KEY\", \"ANTHROPIC_API_KEY\",\n \"FIREWORKS_API_KEY\", \"ZAI_API_KEY\", \"TOGETHER_API_KEY\",\n \"DEEPINFRA_API_KEY\", \"COHERE_API_KEY\", \"XAI_API_KEY\",\n]\nNATIVE_MODEL_PREFIXES = (\"ollama/\", \"lmstudio/\", \"openai-local/\")\n\n\ndef _is_native_model(model):\n return bool(model) and str(model).lower().startswith(NATIVE_MODEL_PREFIXES)\n\n\ndef _local_http_base(value, default):\n base = (value or default).strip().rstrip(\"/\")\n if \"://\" not in base:\n base = \"http://\" + base\n return base\n\n\ndef _api_http_base(value, default):\n return _local_http_base(value, default)\n\n\ndef _find_config():\n if os.path.isfile(CONFIG_PATH):\n return CONFIG_PATH\n if os.path.isfile(CONFIG_PATH_LOCAL):\n return CONFIG_PATH_LOCAL\n return CONFIG_PATH\n\n\ndef _ensure_config_dir():\n try:\n os.makedirs(_APPDATA_DIR, exist_ok=True)\n except Exception:\n pass\n\n\ndef save_config(config):\n _ensure_config_dir()\n tmp = CONFIG_PATH + \".tmp\"\n try:\n with open(tmp, \"w\") as f:\n json.dump(config, f)\n f.flush()\n os.fsync(f.fileno())\n os.replace(tmp, CONFIG_PATH)\n except Exception:\n with open(CONFIG_PATH, \"w\") as f:\n json.dump(config, f)\n\n\ndef load_config(require_keys=True):\n keys = {\n \"CLOUDFLARE_API_KEY\": os.environ.get(\"CLOUDFLARE_API_KEY\"),\n \"AZUREAI_API_KEY\": os.environ.get(\"AZUREAI_API_KEY\"),\n \"AZUREAI_ENDPOINT\": os.environ.get(\"AZUREAI_ENDPOINT\"),\n \"GROQ_API_KEY\": os.environ.get(\"GROQ_API_KEY\"),\n \"HF_TOKEN\": os.environ.get(\"HF_TOKEN\"),\n \"MISTRAL_API_KEY\": os.environ.get(\"MISTRAL_API_KEY\"),\n \"GEMINI_API_KEY\": os.environ.get(\"GEMINI_API_KEY\"),\n \"OPENROUTER_API_KEY\": os.environ.get(\"OPENROUTER_API_KEY\"),\n \"OPENAI_API_KEY\": os.environ.get(\"OPENAI_API_KEY\"),\n \"ANTHROPIC_API_KEY\": os.environ.get(\"ANTHROPIC_API_KEY\"),\n \"FIREWORKS_API_KEY\": os.environ.get(\"FIREWORKS_API_KEY\"),\n \"ZAI_API_KEY\": os.environ.get(\"ZAI_API_KEY\"),\n \"TOGETHER_API_KEY\": os.environ.get(\"TOGETHER_API_KEY\"),\n \"DEEPINFRA_API_KEY\": os.environ.get(\"DEEPINFRA_API_KEY\"),\n \"COHERE_API_KEY\": os.environ.get(\"COHERE_API_KEY\"),\n \"XAI_API_KEY\": os.environ.get(\"XAI_API_KEY\"),\n \"DEFAULT_MODEL\": os.environ.get(\"VISION_MODEL\"),\n }\n cfg_path = _find_config()\n if os.path.isfile(cfg_path):\n try:\n with open(cfg_path, \"r\") as f:\n cfg = json.load(f)\n except (json.JSONDecodeError, IOError):\n cfg = None\n if isinstance(cfg, dict):\n for k in list(keys):\n if not keys[k]:\n keys[k] = cfg.get(k)\n present = [k for k in ALL_PROVIDER_KEYS if keys.get(k)]\n native_model = keys.get(\"DEFAULT_MODEL\", \"\")\n if not present and _is_native_model(native_model):\n present.append(\"DEFAULT_MODEL\")\n if not present and require_keys:\n raise RuntimeError(\n \"No API keys configured.\\n\"\n \" Run setup.py to configure: python setup.py\\n\"\n \" Or set environment variables (any one is enough):\\n\"\n \" $env:GEMINI_API_KEY='...' (aistudio.google.com/apikey)\\n\"\n \" $env:CLOUDFLARE_API_KEY='cfut_...' (cloudflare.com, Workers AI)\\n\"\n \" $env:AZUREAI_API_KEY='...' (Azure AI Foundry)\\n\"\n \" $env:AZUREAI_ENDPOINT='https://...' (Azure AI Foundry endpoint)\\n\"\n \" $env:MISTRAL_API_KEY='...' (console.mistral.ai/api-keys)\\n\"\n \" $env:GROQ_API_KEY='gsk_...' (groq.com, free tier)\\n\"\n \" $env:HF_TOKEN='hf_...' (huggingface.co/settings/tokens)\\n\"\n \" $env:OPENROUTER_API_KEY='sk-or-...' (openrouter.ai/keys)\\n\"\n \" $env:OPENAI_API_KEY='sk-...' (platform.openai.com/api-keys)\\n\"\n \" $env:ANTHROPIC_API_KEY='sk-ant-...' (console.anthropic.com/settings/keys)\\n\"\n \" $env:TOGETHER_API_KEY='...' (api.together.ai)\\n\"\n \" $env:DEEPINFRA_API_KEY='...' (deepinfra.com)\\n\"\n \" $env:COHERE_API_KEY='...' (dashboard.cohere.com/api-keys)\\n\"\n \" $env:XAI_API_KEY='...' (console.x.ai)\\n\"\n \" $env:FIREWORKS_API_KEY='fw_...' (fireworks.ai/api-keys)\\n\"\n \" $env:ZAI_API_KEY='...' (z.ai, Zhipu AI)\\n\"\n \" $env:VISION_MODEL='model-name' (optional default model)\"\n )\n return keys\n\n\nCFG = None\n_BACKEND_MEMORY_LOCK = threading.Lock()\n_CAPABILITY_REFRESH_THREAD = None\n_CAPABILITY_REFRESH_THREAD_LOCK = threading.Lock()\n\n\ndef _load_backend_memory():\n try:\n with open(BACKEND_MEMORY_PATH, \"r\") as f:\n data = json.load(f)\n if isinstance(data, dict) and isinstance(data.get(\"models\"), dict):\n return data\n except (json.JSONDecodeError, IOError, OSError):\n pass\n return {\"version\": 1, \"cooldown_seconds\": BACKEND_LIMIT_COOLDOWN, \"models\": {}}\n\n\ndef _save_backend_memory(memory):\n _ensure_config_dir()\n memory[\"version\"] = 1\n memory[\"cooldown_seconds\"] = BACKEND_LIMIT_COOLDOWN\n tmp = BACKEND_MEMORY_PATH + \".tmp\"\n try:\n with open(tmp, \"w\") as f:\n json.dump(memory, f, indent=2, sort_keys=True)\n f.flush()\n os.fsync(f.fileno())\n os.replace(tmp, BACKEND_MEMORY_PATH)\n except Exception:\n try:\n with open(BACKEND_MEMORY_PATH, \"w\") as f:\n json.dump(memory, f, indent=2, sort_keys=True)\n except Exception:\n pass\n\n\ndef _backend_memory_get(memory, name):\n models = memory.setdefault(\"models\", {})\n entry = models.get(name)\n return entry if isinstance(entry, dict) else {}\n\n\ndef _is_backend_limited(entry, now=None):\n if not entry or entry.get(\"status\") != \"limited\":\n return False\n retry_after = entry.get(\"retry_after\", 0)\n try:\n retry_after = float(retry_after)\n except (TypeError, ValueError):\n retry_after = 0\n return retry_after > (now or time.time())\n\n\ndef _is_limit_error(exc, msg):\n if getattr(exc, \"code\", None) == 429:\n return True\n text = (msg or str(exc) or \"\").lower()\n markers = (\n \"429\",\n \"quota\",\n \"rate limit\",\n \"rate_limit\",\n \"rate-limit\",\n \"too many requests\",\n \"resource exhausted\",\n \"limit exceeded\",\n \"exceeded your current quota\",\n \"insufficient_quota\",\n \"token limit\",\n \"tokens\",\n \"out of token\",\n \"out of tokens\",\n )\n return any(marker in text for marker in markers)\n\n\ndef _is_payment_or_plan_error(exc, msg):\n if getattr(exc, \"code\", None) in (401, 402, 403):\n return True\n text = (msg or str(exc) or \"\").lower()\n markers = (\n \"402\",\n \"payment required\",\n \"billing\",\n \"credits\",\n \"credit balance\",\n \"insufficient credits\",\n \"insufficient balance\",\n \"upgrade\",\n \"subscribe\",\n \"subscription\",\n \"paid plan\",\n \"requires payment\",\n \"not enough funds\",\n \"prepaid\",\n \"trial\",\n \"free tier\",\n \"not allowed\",\n \"forbidden\",\n )\n return any(marker in text for marker in markers)\n\n\ndef _looks_free_or_included_backend(name):\n text = (name or \"\").lower()\n markers = (\n \":free\",\n \"ollama\",\n \"lm studio\",\n \"lmstudio\",\n \"openai-local\",\n \"flash-lite\",\n \"mini\",\n \"nano\",\n \"free\",\n )\n return any(marker in text for marker in markers)\n\n\ndef _infer_access_tier(results):\n successes = [r for r in results if r.get(\"status\") == \"ok\"]\n paid_success = [r for r in successes if r.get(\"access_kind\") == \"paid_or_metered\"]\n local_success = [r for r in successes if r.get(\"access_kind\") == \"local\"]\n free_success = [r for r in successes if r.get(\"access_kind\") == \"free_or_included\"]\n plan_errors = [r for r in results if r.get(\"status\") == \"payment_or_plan_limited\"]\n if paid_success:\n return \"paid_available\"\n if free_success and plan_errors:\n return \"free_only\"\n if free_success:\n return \"free_or_included_available\"\n if local_success and len(local_success) == len(successes):\n return \"local_only\"\n if local_success:\n return \"local_available\"\n if plan_errors:\n return \"free_or_unfunded\"\n return \"unknown\"\n\n\ndef _record_backend_success(memory, name):\n with _BACKEND_MEMORY_LOCK:\n memory.setdefault(\"models\", {})[name] = {\n \"status\": \"ok\",\n \"last_success\": int(time.time()),\n \"last_error\": \"\",\n \"retry_after\": 0,\n }\n _save_backend_memory(memory)\n\n\ndef _record_backend_failure(memory, name, exc, msg):\n with _BACKEND_MEMORY_LOCK:\n now = int(time.time())\n limited = _is_limit_error(exc, msg)\n entry = {\n \"status\": \"limited\" if limited else \"failed\",\n \"last_failure\": now,\n \"last_error\": msg,\n \"retry_after\": now + BACKEND_LIMIT_COOLDOWN if limited else 0,\n }\n memory.setdefault(\"models\", {})[name] = entry\n _save_backend_memory(memory)\n\n\ndef _filter_strategies_by_memory(strategies, memory):\n now = time.time()\n available = []\n skipped = []\n for name, fn in strategies:\n entry = _backend_memory_get(memory, name)\n if _is_backend_limited(entry, now):\n skipped.append((name, int(entry.get(\"retry_after\", 0) - now)))\n continue\n available.append((name, fn))\n return available, skipped\n\n\ndef _has_unknown_backend_status(strategies, memory):\n models = memory.get(\"models\", {})\n return any(name not in models for name, _ in strategies)\n\n\ndef _capability_profile_due(memory, now=None, force=False):\n if force:\n return True\n profile = memory.get(\"capability_profile\", {})\n if not isinstance(profile, dict):\n return True\n if profile.get(\"status\") == \"running\":\n started = int(profile.get(\"refresh_started_at\") or 0)\n if started and int(now or time.time()) - started < CAPABILITY_REFRESH_STALE_AFTER:\n return False\n next_refresh = int(profile.get(\"next_refresh\") or 0)\n return next_refresh <= int(now or time.time())\n\n\ndef _mark_capability_refresh_started(force=False, reason=\"startup\"):\n now = int(time.time())\n with _BACKEND_MEMORY_LOCK:\n memory = _load_backend_memory()\n if not _capability_profile_due(memory, now, force=force):\n return False\n profile = memory.setdefault(\"capability_profile\", {})\n profile.update({\n \"status\": \"running\",\n \"reason\": reason,\n \"refresh_started_at\": now,\n \"last_error\": \"\",\n })\n _save_backend_memory(memory)\n return True\n\n\ndef _finish_capability_refresh(profile_update):\n now = int(time.time())\n with _BACKEND_MEMORY_LOCK:\n memory = _load_backend_memory()\n profile = memory.setdefault(\"capability_profile\", {})\n profile.update(profile_update)\n profile[\"last_refresh\"] = now\n profile[\"next_refresh\"] = now + CAPABILITY_REFRESH_INTERVAL\n profile[\"refresh_started_at\"] = 0\n _save_backend_memory(memory)\n\n\ndef _strategy_access_kind(name):\n text = (name or \"\").lower()\n if \"ollama\" in text or \"lm studio\" in text or \"lmstudio\" in text or \"openai-local\" in text:\n return \"local\"\n if _looks_free_or_included_backend(name):\n return \"free_or_included\"\n return \"paid_or_metered\"\n\n\ndef _provider_label_from_strategy(name):\n text = (name or \"\").lower()\n providers = (\n (\"openrouter\", \"OpenRouter\"),\n (\"gemini\", \"Gemini\"),\n (\"openai\", \"OpenAI\"),\n (\"anthropic\", \"Anthropic\"),\n (\"claude\", \"Anthropic\"),\n (\"together\", \"Together\"),\n (\"deepinfra\", \"DeepInfra\"),\n (\"cohere\", \"Cohere\"),\n (\"xai\", \"xAI\"),\n (\"grok\", \"xAI\"),\n (\"cloudflare\", \"Cloudflare\"),\n (\"azure\", \"Azure AI\"),\n (\"groq\", \"Groq\"),\n (\"hf \", \"HuggingFace\"),\n (\"hugging\", \"HuggingFace\"),\n (\"mistral\", \"Mistral\"),\n (\"fireworks\", \"Fireworks\"),\n (\"zai\", \"ZAI\"),\n (\"ollama\", \"Ollama\"),\n (\"lm studio\", \"LM Studio\"),\n (\"lmstudio\", \"LM Studio\"),\n )\n for marker, label in providers:\n if marker in text:\n return label\n return \"unknown\"\n\n\ndef _dedupe_strategies(strategies):\n seen = set()\n deduped = []\n for name, fn in strategies:\n if name in seen:\n continue\n seen.add(name)\n deduped.append((name, fn))\n return deduped\n\n\ndef _try_detect_new_default_backend():\n if os.environ.get(\"VISION_TOOL_SKIP_REFRESH_DETECT\"):\n return None\n try:\n import importlib\n setup_mod = importlib.import_module(\"setup\")\n except Exception:\n return None\n for fn_name in (\"detect_native_vision_model\", \"detect_existing_cloud_vision_model\"):\n fn = getattr(setup_mod, fn_name, None)\n if not callable(fn):\n continue\n try:\n found = fn()\n except Exception:\n found = None\n if found and found.get(\"default_model\"):\n try:\n save_fn = getattr(setup_mod, \"save_detected_vision_backend\", None)\n if callable(save_fn):\n save_fn(found[\"default_model\"], found.get(\"config\", {}))\n except Exception:\n pass\n return found\n return None\n\n\ndef _build_capability_probe_strategies():\n global CFG\n CFG = load_config()\n prompt = \"Reply with one short word describing this image.\"\n strategies = _build_strategies(\"img\", TINY_PNG_B64, \"image/png\", prompt=prompt)\n model = CFG.get(\"DEFAULT_MODEL\", \"\") or None\n if model:\n _insert_model_strategies(strategies, model, \"img\", TINY_PNG_B64, \"image/png\", prompt=prompt)\n return _dedupe_strategies([(n, f) for n, f in strategies if _has_key(n)])\n\n\ndef refresh_capability_profile(force=False, reason=\"manual\", _already_marked=False):\n if os.environ.get(\"VISION_TOOL_DISABLE_CAPABILITY_REFRESH\"):\n return False\n if not _already_marked and not _mark_capability_refresh_started(force=force, reason=reason):\n return False\n\n results = []\n detected = None\n status = \"ok\"\n last_error = \"\"\n try:\n detected = _try_detect_new_default_backend()\n strategies = _build_capability_probe_strategies()\n memory = _load_backend_memory()\n for name, fn in strategies:\n item = {\n \"name\": name,\n \"provider\": _provider_label_from_strategy(name),\n \"access_kind\": _strategy_access_kind(name),\n \"status\": \"unknown\",\n }\n try:\n text = _call_with_timeout(fn, CAPABILITY_PROBE_TIMEOUT)\n if text and str(text).strip():\n item[\"status\"] = \"ok\"\n _record_backend_success(memory, name)\n else:\n item[\"status\"] = \"failed\"\n item[\"error\"] = \"empty response\"\n except Exception as exc:\n msg = str(exc)\n if hasattr(exc, \"code\"):\n msg = f\"HTTP {exc.code}\"\n item[\"error\"] = msg[:300]\n if _is_payment_or_plan_error(exc, msg):\n item[\"status\"] = \"payment_or_plan_limited\"\n elif _is_limit_error(exc, msg):\n item[\"status\"] = \"quota_or_rate_limited\"\n else:\n item[\"status\"] = \"failed\"\n _record_backend_failure(memory, name, exc, msg)\n results.append(item)\n except Exception as exc:\n status = \"error\"\n last_error = str(exc)[:500]\n\n providers = {}\n for item in results:\n provider = item.get(\"provider\") or \"unknown\"\n entry = providers.setdefault(provider, {\"ok\": 0, \"failed\": 0, \"limited\": 0, \"paid_ok\": 0, \"free_ok\": 0})\n if item.get(\"status\") == \"ok\":\n entry[\"ok\"] += 1\n if item.get(\"access_kind\") == \"paid_or_metered\":\n entry[\"paid_ok\"] += 1\n if item.get(\"access_kind\") in (\"free_or_included\", \"local\"):\n entry[\"free_ok\"] += 1\n elif item.get(\"status\") in (\"quota_or_rate_limited\", \"payment_or_plan_limited\"):\n entry[\"limited\"] += 1\n else:\n entry[\"failed\"] += 1\n\n access_tier = _infer_access_tier(results)\n update = {\n \"version\": 1,\n \"status\": status,\n \"last_error\": last_error,\n \"access_tier\": access_tier,\n \"refresh_interval_seconds\": CAPABILITY_REFRESH_INTERVAL,\n \"default_model\": (CFG or {}).get(\"DEFAULT_MODEL\", \"\") if isinstance(CFG, dict) else \"\",\n \"detected_backend\": detected or {},\n \"providers\": providers,\n \"results\": results,\n }\n _finish_capability_refresh(update)\n return True\n\n\ndef start_background_capability_refresh(force=False, reason=\"startup\"):\n if os.environ.get(\"VISION_TOOL_DISABLE_CAPABILITY_REFRESH\"):\n return False\n global _CAPABILITY_REFRESH_THREAD\n with _CAPABILITY_REFRESH_THREAD_LOCK:\n if _CAPABILITY_REFRESH_THREAD and _CAPABILITY_REFRESH_THREAD.is_alive():\n return False\n if not _mark_capability_refresh_started(force=force, reason=reason):\n return False\n\n def worker():\n try:\n refresh_capability_profile(force=True, reason=reason, _already_marked=True)\n except Exception as exc:\n _finish_capability_refresh({\"status\": \"error\", \"last_error\": str(exc)[:500]})\n\n _CAPABILITY_REFRESH_THREAD = threading.Thread(\n target=worker,\n name=\"vision-tool-capability-refresh\",\n daemon=True,\n )\n _CAPABILITY_REFRESH_THREAD.start()\n return True\n\n\ndef _has_key(name):\n if CFG is None:\n return True\n if \"Ollama\" in name or \"ollama\" in name or \"LM Studio\" in name or \"lmstudio\" in name:\n return True\n if \"OpenAI\" in name or \"openai\" in name:\n return bool(CFG.get(\"OPENAI_API_KEY\"))\n if \"Anthropic\" in name or \"Claude\" in name or \"anthropic\" in name:\n return bool(CFG.get(\"ANTHROPIC_API_KEY\"))\n if \"Together\" in name or \"together\" in name:\n return bool(CFG.get(\"TOGETHER_API_KEY\"))\n if \"DeepInfra\" in name or \"deepinfra\" in name:\n return bool(CFG.get(\"DEEPINFRA_API_KEY\"))\n if \"Cohere\" in name or \"cohere\" in name:\n return bool(CFG.get(\"COHERE_API_KEY\"))\n if \"xAI\" in name or \"XAI\" in name or \"Grok\" in name or \"xai\" in name:\n return bool(CFG.get(\"XAI_API_KEY\"))\n if \"OpenRouter\" in name or \"openrouter\" in name:\n return bool(CFG.get(\"OPENROUTER_API_KEY\"))\n if \"Cloudflare\" in name or \"cloudflare\" in name or \"CF \" in name:\n return bool(CFG.get(\"CLOUDFLARE_API_KEY\"))\n if \"Azure\" in name or \"azure\" in name or \"azureai\" in name:\n return bool(CFG.get(\"AZUREAI_API_KEY\")) and bool(CFG.get(\"AZUREAI_ENDPOINT\"))\n if \"Groq\" in name or \"groq\" in name:\n return bool(CFG.get(\"GROQ_API_KEY\"))\n if \"HF\" in name or \"Hugging\" in name or \"HuggingFace\" in name:\n return bool(CFG.get(\"HF_TOKEN\"))\n if \"Mistral\" in name or \"mistral\" in name:\n return bool(CFG.get(\"MISTRAL_API_KEY\"))\n if \"Fireworks\" in name or \"fireworks\" in name:\n return bool(CFG.get(\"FIREWORKS_API_KEY\"))\n if \"Gemini\" in name or \"gemini\" in name or \"Google\" in name:\n return bool(CFG.get(\"GEMINI_API_KEY\"))\n if \"Zai\" in name or \"zai\" in name or \"Z.AI\" in name or \"ZAI\" in name:\n return bool(CFG.get(\"ZAI_API_KEY\"))\n return False\n\n\ndef _print_available_keys():\n key_labels = [\n (\"CLOUDFLARE_API_KEY\", \"Cloudflare\"),\n (\"AZUREAI_API_KEY\", \"Azure AI Foundry\"),\n (\"GROQ_API_KEY\", \"Groq\"),\n (\"HF_TOKEN\", \"HuggingFace\"),\n (\"MISTRAL_API_KEY\", \"Mistral AI\"),\n (\"FIREWORKS_API_KEY\", \"Fireworks AI\"),\n (\"ZAI_API_KEY\", \"Zhipu AI\"),\n (\"GEMINI_API_KEY\", \"Google Gemini\"),\n (\"OPENROUTER_API_KEY\", \"OpenRouter\"),\n (\"OPENAI_API_KEY\", \"OpenAI\"),\n (\"ANTHROPIC_API_KEY\", \"Anthropic\"),\n (\"TOGETHER_API_KEY\", \"Together AI\"),\n (\"DEEPINFRA_API_KEY\", \"DeepInfra\"),\n (\"COHERE_API_KEY\", \"Cohere\"),\n (\"XAI_API_KEY\", \"xAI\"),\n ]\n parts = []\n for env_key, label in key_labels:\n if CFG and CFG.get(env_key):\n parts.append(f\"{label} \\u2713\")\n else:\n parts.append(f\"{label} \\u2717\")\n print(f\"KEYS: {' '.join(parts)}\", file=sys.stderr, flush=True)\n\n\n# ── File-type helpers ────────────────────────────────────────────────────\nVIDEO_EXT = {\".mp4\", \".webm\", \".mov\", \".avi\", \".mkv\", \".flv\", \".wmv\", \".m4v\"}\nIMAGE_EXT = {\".png\", \".jpg\", \".jpeg\", \".webp\", \".bmp\"}\n\n\ndef get_mime(path):\n m, _ = mimetypes.guess_type(path)\n if m:\n return m\n ext = os.path.splitext(path)[1].lower()\n img = {\".png\": \"image/png\", \".jpg\": \"image/jpeg\", \".jpeg\": \"image/jpeg\",\n \".webp\": \"image/webp\", \".bmp\": \"image/bmp\"}\n vid = {\".mp4\": \"video/mp4\", \".webm\": \"video/webm\", \".mov\": \"video/quicktime\",\n \".avi\": \"video/x-msvideo\", \".mkv\": \"video/x-matroska\",\n \".flv\": \"video/x-flv\", \".wmv\": \"video/x-ms-wmv\", \".m4v\": \"video/mp4\"}\n return img.get(ext) or vid.get(ext) or \"image/png\"\n\n\ndef is_video(path):\n return os.path.splitext(path)[1].lower() in VIDEO_EXT\n\n\ndef is_image(path):\n return os.path.splitext(path)[1].lower() in IMAGE_EXT\n\n\n# ── Image resize ─────────────────────────────────────────────────────────\nMAX_IMAGE_DIM = 2048\n\n\ndef resize_image(path, max_dim=None):\n if isinstance(path, bool) or not isinstance(path, (str, bytes, os.PathLike)):\n raise TypeError(\"path must be a filesystem path\")\n if max_dim is None:\n max_dim = MAX_IMAGE_DIM\n try:\n from PIL import Image\n img = Image.open(path)\n img.load()\n w, h = img.size\n if w > max_dim or h > max_dim:\n if w > h:\n nw, nh = max_dim, int(h * max_dim / w)\n else:\n nw, nh = int(w * max_dim / h), max_dim\n img = img.resize((nw, nh), Image.LANCZOS)\n buf = io.BytesIO()\n img.save(buf, format=\"JPEG\", quality=75)\n result = buf.getvalue()\n if len(result) > 3_000_000:\n quality = 65\n while len(result) > 3_000_000 and quality >= 15:\n buf = io.BytesIO()\n img.save(buf, format=\"JPEG\", quality=quality)\n result = buf.getvalue()\n quality -= 10\n return result, \"image/jpeg\"\n except Exception:\n with open(path, \"rb\") as f:\n return f.read(), get_mime(path)\n\n\n# ── Video keyframe extraction ────────────────────────────────────────────\ndef extract_video_frames(path, max_frames=8):\n ext = os.path.splitext(path)[1].lower()\n if ext == \".gif\":\n try:\n from PIL import Image\n img = Image.open(path)\n frames = []\n try:\n while True:\n frames.append(img.copy().convert(\"RGB\"))\n img.seek(img.tell() + 1)\n except EOFError:\n pass\n if not frames:\n with open(path, \"rb\") as f:\n return [(f.read(), \"image/gif\")]\n step = max(len(frames) // max_frames, 1)\n selected = frames[::step][:max_frames]\n result = []\n for f in selected:\n buf = io.BytesIO()\n f.save(buf, format=\"JPEG\", quality=85)\n result.append((buf.getvalue(), \"image/jpeg\"))\n return result\n except ImportError:\n with open(path, \"rb\") as f:\n return [(f.read(), \"image/gif\")]\n except Exception:\n with open(path, \"rb\") as f:\n return [(f.read(), \"image/gif\")]\n\n try:\n dur = subprocess.run(\n [\"ffprobe\", \"-v\", \"error\", \"-show_entries\", \"format=duration\",\n \"-of\", \"default=noprint_wrappers=1:nokey=1\", path],\n capture_output=True, text=True, timeout=30,\n )\n duration = float(dur.stdout.strip())\n except Exception:\n duration = 10\n if duration <= 0:\n duration = 10\n num = min(max_frames, max(2, int(duration)))\n interval = duration / num\n tmpdir = tempfile.mkdtemp()\n frames = []\n try:\n for i in range(num):\n ts = i * interval\n out = os.path.join(tmpdir, f\"f_{i:03d}.jpg\")\n subprocess.run(\n [\"ffmpeg\", \"-ss\", str(ts), \"-i\", path,\n \"-vframes\", \"1\", \"-q:v\", \"2\", \"-vf\", \"scale=1024:-1\",\n \"-y\", out],\n capture_output=True, timeout=30,\n )\n if os.path.isfile(out) and os.path.getsize(out) > 0:\n with open(out, \"rb\") as f:\n frames.append((f.read(), \"image/jpeg\"))\n os.remove(out)\n except Exception:\n pass\n finally:\n shutil.rmtree(tmpdir, ignore_errors=True)\n if not frames:\n with open(path, \"rb\") as f:\n return [(f.read(), get_mime(path))]\n return frames\n\n\n# ── API helpers ──────────────────────────────────────────────────────────\ndef b64(data):\n return base64.b64encode(data).decode(\"utf-8\")\n\n\ndef build_multimodal_content(frames, prompt):\n parts = [{\"type\": \"text\", \"text\": prompt}]\n for data, mime in frames:\n parts.append({\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64(data)}\"}})\n return parts\n\n\ndef build_gemini_parts(frames, prompt):\n parts = [{\"text\": prompt}]\n for data, mime in frames:\n parts.append({\"inline_data\": {\"mime_type\": mime, \"data\": b64(data)}})\n return parts\n\n\n# ── Backend callers ──────────────────────────────────────────────────────\n\n# ── Cloudflare Workers AI caller (OpenAI-compatible) ────────────\nOLLAMA_ENDPOINT = _local_http_base(os.environ.get(\"OLLAMA_HOST\"), \"http://127.0.0.1:11434\")\nLMSTUDIO_ENDPOINT = _local_http_base(os.environ.get(\"LMSTUDIO_BASE_URL\"), \"http://127.0.0.1:1234/v1\")\nOPENROUTER_ENDPOINT = _api_http_base(os.environ.get(\"OPENROUTER_BASE_URL\"), \"https://openrouter.ai/api/v1\")\nOPENAI_ENDPOINT = _api_http_base(os.environ.get(\"OPENAI_BASE_URL\"), \"https://api.openai.com/v1\")\nTOGETHER_ENDPOINT = _api_http_base(os.environ.get(\"TOGETHER_BASE_URL\"), \"https://api.together.xyz/v1\")\nDEEPINFRA_ENDPOINT = _api_http_base(os.environ.get(\"DEEPINFRA_BASE_URL\"), \"https://api.deepinfra.com/v1/openai\")\nXAI_ENDPOINT = _api_http_base(os.environ.get(\"XAI_BASE_URL\"), \"https://api.x.ai/v1\")\nANTHROPIC_ENDPOINT = _api_http_base(os.environ.get(\"ANTHROPIC_BASE_URL\"), \"https://api.anthropic.com/v1\")\nCOHERE_ENDPOINT = _api_http_base(os.environ.get(\"COHERE_BASE_URL\"), \"https://api.cohere.com/v2\")\n\n\ndef call_ollama(b64data, mime, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": prompt, \"images\": [b64data]}],\n \"stream\": False,\n }\n req = urllib.request.Request(\n f\"{OLLAMA_ENDPOINT}/api/chat\",\n data=json.dumps(payload).encode(),\n headers={\"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=60)\n data = json.loads(resp.read())\n msg = data.get(\"message\", {})\n return msg.get(\"content\") or data.get(\"response\") or str(data)\n\n\ndef call_ollama_multi(frames, prompt, model):\n images = []\n for data, _ in frames:\n images.append(b64(data) if isinstance(data, bytes) else str(data))\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": prompt, \"images\": images}],\n \"stream\": False,\n }\n req = urllib.request.Request(\n f\"{OLLAMA_ENDPOINT}/api/chat\",\n data=json.dumps(payload).encode(),\n headers={\"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=90)\n data = json.loads(resp.read())\n msg = data.get(\"message\", {})\n return msg.get(\"content\") or data.get(\"response\") or str(data)\n\n\ndef call_lmstudio(b64data, mime, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n \"max_tokens\": 512,\n }\n req = urllib.request.Request(\n f\"{LMSTUDIO_ENDPOINT}/chat/completions\",\n data=json.dumps(payload).encode(),\n headers={\"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=60)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_lmstudio_multi(frames, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n \"max_tokens\": 768,\n }\n req = urllib.request.Request(\n f\"{LMSTUDIO_ENDPOINT}/chat/completions\",\n data=json.dumps(payload).encode(),\n headers={\"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=90)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef _openrouter_headers():\n return {\n \"Authorization\": f\"Bearer {CFG['OPENROUTER_API_KEY']}\",\n \"Content-Type\": \"application/json\",\n \"HTTP-Referer\": \"https://github.com/farhanic017/vision-tool\",\n \"X-Title\": \"vision-tool\",\n }\n\n\ndef call_openrouter(b64data, mime, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n f\"{OPENROUTER_ENDPOINT}/chat/completions\",\n data=json.dumps(payload).encode(),\n headers=_openrouter_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=60)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_openrouter_multi(frames, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n f\"{OPENROUTER_ENDPOINT}/chat/completions\",\n data=json.dumps(payload).encode(),\n headers=_openrouter_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=90)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef _extract_chat_completion(data):\n choice = (data.get(\"choices\") or [{}])[0] if isinstance(data, dict) else {}\n msg = choice.get(\"message\", {}) if isinstance(choice, dict) else {}\n content = msg.get(\"content\", \"\")\n if isinstance(content, list):\n parts = []\n for item in content:\n if isinstance(item, dict):\n parts.append(str(item.get(\"text\") or item.get(\"content\") or \"\"))\n elif item:\n parts.append(str(item))\n return \" \".join(parts).strip()\n return str(content or \"\").strip()\n\n\ndef _call_openai_compatible(endpoint, key_name, b64data, mime, prompt, model, timeout=60):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n f\"{endpoint}/chat/completions\",\n data=json.dumps(payload).encode(),\n headers={\"Authorization\": f\"Bearer {CFG[key_name]}\", \"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=timeout)\n return _extract_chat_completion(json.loads(resp.read()))\n\n\ndef _call_openai_compatible_multi(endpoint, key_name, frames, prompt, model, timeout=90):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n f\"{endpoint}/chat/completions\",\n data=json.dumps(payload).encode(),\n headers={\"Authorization\": f\"Bearer {CFG[key_name]}\", \"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=timeout)\n return _extract_chat_completion(json.loads(resp.read()))\n\n\ndef call_openai(b64data, mime, prompt, model=\"gpt-4o-mini\"):\n return _call_openai_compatible(OPENAI_ENDPOINT, \"OPENAI_API_KEY\", b64data, mime, prompt, model)\n\n\ndef call_openai_multi(frames, prompt, model=\"gpt-4o-mini\"):\n return _call_openai_compatible_multi(OPENAI_ENDPOINT, \"OPENAI_API_KEY\", frames, prompt, model)\n\n\ndef call_together(b64data, mime, prompt, model=\"moonshotai/Kimi-K2.5\"):\n return _call_openai_compatible(TOGETHER_ENDPOINT, \"TOGETHER_API_KEY\", b64data, mime, prompt, model)\n\n\ndef call_together_multi(frames, prompt, model=\"moonshotai/Kimi-K2.5\"):\n return _call_openai_compatible_multi(TOGETHER_ENDPOINT, \"TOGETHER_API_KEY\", frames, prompt, model)\n\n\ndef call_deepinfra(b64data, mime, prompt, model=\"Qwen/Qwen2.5-VL-72B-Instruct\"):\n return _call_openai_compatible(DEEPINFRA_ENDPOINT, \"DEEPINFRA_API_KEY\", b64data, mime, prompt, model)\n\n\ndef call_deepinfra_multi(frames, prompt, model=\"Qwen/Qwen2.5-VL-72B-Instruct\"):\n return _call_openai_compatible_multi(DEEPINFRA_ENDPOINT, \"DEEPINFRA_API_KEY\", frames, prompt, model)\n\n\ndef call_xai(b64data, mime, prompt, model=\"grok-4.3\"):\n return _call_openai_compatible(XAI_ENDPOINT, \"XAI_API_KEY\", b64data, mime, prompt, model)\n\n\ndef call_xai_multi(frames, prompt, model=\"grok-4.3\"):\n return _call_openai_compatible_multi(XAI_ENDPOINT, \"XAI_API_KEY\", frames, prompt, model)\n\n\ndef _extract_anthropic_text(data):\n parts = []\n for item in data.get(\"content\", []) if isinstance(data, dict) else []:\n if isinstance(item, dict):\n parts.append(str(item.get(\"text\") or \"\"))\n return \" \".join(parts).strip()\n\n\ndef call_anthropic(b64data, mime, prompt, model=\"claude-sonnet-4-5\"):\n payload = {\n \"model\": model,\n \"max_tokens\": 2048,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image\", \"source\": {\"type\": \"base64\", \"media_type\": mime, \"data\": b64data}},\n ]}],\n }\n req = urllib.request.Request(\n f\"{ANTHROPIC_ENDPOINT}/messages\",\n data=json.dumps(payload).encode(),\n headers={\n \"x-api-key\": CFG[\"ANTHROPIC_API_KEY\"],\n \"anthropic-version\": \"2023-06-01\",\n \"Content-Type\": \"application/json\",\n },\n )\n resp = urllib.request.urlopen(req, timeout=60)\n return _extract_anthropic_text(json.loads(resp.read()))\n\n\ndef call_anthropic_multi(frames, prompt, model=\"claude-sonnet-4-5\"):\n content = [{\"type\": \"text\", \"text\": prompt}]\n for data, mime in frames:\n content.append({\n \"type\": \"image\",\n \"source\": {\"type\": \"base64\", \"media_type\": mime, \"data\": b64(data) if isinstance(data, bytes) else data},\n })\n payload = {\"model\": model, \"max_tokens\": 2048, \"messages\": [{\"role\": \"user\", \"content\": content}]}\n req = urllib.request.Request(\n f\"{ANTHROPIC_ENDPOINT}/messages\",\n data=json.dumps(payload).encode(),\n headers={\n \"x-api-key\": CFG[\"ANTHROPIC_API_KEY\"],\n \"anthropic-version\": \"2023-06-01\",\n \"Content-Type\": \"application/json\",\n },\n )\n resp = urllib.request.urlopen(req, timeout=90)\n return _extract_anthropic_text(json.loads(resp.read()))\n\n\ndef _extract_cohere_text(data):\n if not isinstance(data, dict):\n return \"\"\n message = data.get(\"message\", {})\n content = message.get(\"content\", []) if isinstance(message, dict) else data.get(\"content\", [])\n if isinstance(content, str):\n return content.strip()\n parts = []\n for item in content or []:\n if isinstance(item, dict):\n parts.append(str(item.get(\"text\") or item.get(\"content\") or \"\"))\n elif item:\n parts.append(str(item))\n return \" \".join(parts).strip()\n\n\ndef call_cohere(b64data, mime, prompt, model=\"command-a-vision-07-2025\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n }\n req = urllib.request.Request(\n f\"{COHERE_ENDPOINT}/chat\",\n data=json.dumps(payload).encode(),\n headers={\"Authorization\": f\"Bearer {CFG['COHERE_API_KEY']}\", \"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=60)\n return _extract_cohere_text(json.loads(resp.read()))\n\n\ndef call_cohere_multi(frames, prompt, model=\"command-a-vision-07-2025\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n }\n req = urllib.request.Request(\n f\"{COHERE_ENDPOINT}/chat\",\n data=json.dumps(payload).encode(),\n headers={\"Authorization\": f\"Bearer {CFG['COHERE_API_KEY']}\", \"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=90)\n return _extract_cohere_text(json.loads(resp.read()))\n\n\nCLOUDFLARE_ACCOUNT_ID = \"c782ccfebd6eb876a9ef860d61588da7\"\nCLOUDFLARE_ENDPOINT = f\"https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1/chat/completions\"\n\n\ndef _cloudflare_headers():\n return {\n \"Authorization\": f\"Bearer {CFG['CLOUDFLARE_API_KEY']}\",\n \"Content-Type\": \"application/json\",\n }\n\n\ndef _cloudflare_extract(response):\n msg = response[\"choices\"][0][\"message\"]\n content = msg.get(\"content\")\n if content:\n return content\n reasoning = msg.get(\"reasoning\") or msg.get(\"reasoning_content\")\n if reasoning:\n return reasoning\n return str(msg)\n\n\ndef call_cloudflare(b64data, mime, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n CLOUDFLARE_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_cloudflare_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return _cloudflare_extract(json.loads(resp.read()))\n\n\ndef call_cloudflare_multi(frames, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n CLOUDFLARE_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_cloudflare_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return _cloudflare_extract(json.loads(resp.read()))\n\n\n# ── Azure AI Foundry caller (OpenAI-compatible) ────────────────────\nAZUREAI_DEFAULT_DEPLOYMENT = \"Phi-4-multimodal-instruct\"\nAZUREAI_API_VERSION = \"2024-10-21\"\nAZUREAI_API_VERSION_NEW = \"2025-04-01-preview\"\n\n# GPT-5.x models don't support max_tokens parameter\n_AZUREAI_NO_MAX_TOKENS = {\"gpt-5.4\", \"gpt-5.4-mini\", \"gpt-5.4-nano\", \"gpt-5.1\"}\n# o4-mini needs newer API version\n_AZUREAI_NEW_API_VERSION = {\"o4-mini\"}\n\n\ndef _azureai_endpoint(model):\n base = (CFG.get(\"AZUREAI_ENDPOINT\") or \"\").rstrip(\"/\")\n api_ver = AZUREAI_API_VERSION\n if model in _AZUREAI_NEW_API_VERSION:\n api_ver = AZUREAI_API_VERSION_NEW\n if \"/openai/deployments/\" in base:\n return f\"{base}/chat/completions?api-version={api_ver}\"\n return f\"{base}/openai/deployments/{model}/chat/completions?api-version={api_ver}\"\n\n\ndef _azureai_headers():\n return {\n \"api-key\": CFG[\"AZUREAI_API_KEY\"],\n \"Content-Type\": \"application/json\",\n }\n\n\ndef call_azureai(b64data, mime, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n }\n if model not in _AZUREAI_NO_MAX_TOKENS:\n payload[\"max_tokens\"] = 2048\n req = urllib.request.Request(\n _azureai_endpoint(model),\n data=json.dumps(payload).encode(),\n headers=_azureai_headers(),\n )\n try:\n resp = urllib.request.urlopen(req, timeout=30)\n except urllib.error.HTTPError as e:\n if e.code == 400:\n base = CFG.get(\"AZUREAI_ENDPOINT\", \"\")\n if \"services.ai.azure.com\" in base:\n raise RuntimeError(\n f\"Azure endpoint uses 'services.ai.azure.com' but needs an Azure OpenAI \"\n f\"endpoint (https://{{name}}.openai.azure.com). Got: {base}\"\n )\n raise\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_azureai_multi(frames, prompt, model):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n }\n if model not in _AZUREAI_NO_MAX_TOKENS:\n payload[\"max_tokens\"] = 2048\n req = urllib.request.Request(\n _azureai_endpoint(model),\n data=json.dumps(payload).encode(),\n headers=_azureai_headers(),\n )\n try:\n resp = urllib.request.urlopen(req, timeout=30)\n except urllib.error.HTTPError as e:\n if e.code == 400:\n base = CFG.get(\"AZUREAI_ENDPOINT\", \"\")\n if \"services.ai.azure.com\" in base:\n raise RuntimeError(\n f\"Azure endpoint uses 'services.ai.azure.com' but needs an Azure OpenAI \"\n f\"endpoint (https://{{name}}.openai.azure.com). Got: {base}\"\n )\n raise\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\n# ── Groq caller (OpenAI-compatible) ──────────────────────────────────\nGROQ_ENDPOINT = \"https://api.groq.com/openai/v1/chat/completions\"\n\n\ndef _groq_headers():\n return {\n \"Authorization\": f\"Bearer {CFG['GROQ_API_KEY']}\",\n \"Content-Type\": \"application/json\",\n \"User-Agent\": \"vision-tool/1.0\",\n }\n\n\ndef call_groq(b64data, mime, prompt, model=\"meta-llama/llama-4-scout-17b-16e-instruct\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n }\n req = urllib.request.Request(\n GROQ_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_groq_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=15)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_groq_multi(frames, prompt, model=\"meta-llama/llama-4-scout-17b-16e-instruct\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n }\n req = urllib.request.Request(\n GROQ_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_groq_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=15)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\n# ── Hugging Face Inference Providers caller ───────────────────────────\nHF_ROUTER_ENDPOINT = \"https://router.huggingface.co/v1/chat/completions\"\n\n\ndef _hf_headers():\n return {\n \"Authorization\": f\"Bearer {CFG['HF_TOKEN']}\",\n \"Content-Type\": \"application/json\",\n }\n\n\ndef _hf_default_model():\n return \"Qwen/Qwen3-VL-8B-Instruct\"\n\n\ndef call_hf_inference(b64data, mime, prompt, model=None):\n model = model or _hf_default_model()\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n {\"type\": \"text\", \"text\": prompt},\n ]}],\n \"max_tokens\": 1024,\n }\n req = urllib.request.Request(\n HF_ROUTER_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_hf_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=60)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_hf_multi(frames, prompt, model=None):\n model = model or _hf_default_model()\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n \"max_tokens\": 1024,\n }\n req = urllib.request.Request(\n HF_ROUTER_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_hf_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=60)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\n# ── Mistral AI caller (OpenAI-compatible) ───────────────────────────\nMISTRAL_ENDPOINT = \"https://api.mistral.ai/v1/chat/completions\"\n\n\ndef _mistral_headers():\n return {\n \"Authorization\": f\"Bearer {CFG['MISTRAL_API_KEY']}\",\n \"Content-Type\": \"application/json\",\n }\n\n\ndef call_mistral(b64data, mime, prompt, model=\"pixtral-large-latest\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n }\n req = urllib.request.Request(\n MISTRAL_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_mistral_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_mistral_multi(frames, prompt, model=\"pixtral-large-latest\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n }\n req = urllib.request.Request(\n MISTRAL_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_mistral_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\n# ── Fireworks AI caller (OpenAI-compatible) ─────────────────────────\nFIREWORKS_ENDPOINT = \"https://api.fireworks.ai/inference/v1/chat/completions\"\n\n\ndef _fireworks_headers():\n return {\n \"Authorization\": f\"Bearer {CFG['FIREWORKS_API_KEY']}\",\n \"Content-Type\": \"application/json\",\n }\n\n\ndef call_fireworks(b64data, mime, prompt, model=\"accounts/fireworks/models/llama-v3p2-90b-vision-instruct\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n FIREWORKS_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_fireworks_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_fireworks_multi(frames, prompt, model=\"accounts/fireworks/models/llama-v3p2-90b-vision-instruct\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n FIREWORKS_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_fireworks_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\n# ── Google Gemini caller ────────────────────────────────────────────\nGEMINI_BASE = \"https://generativelanguage.googleapis.com/v1beta/models\"\n\n\ndef _gemini_url(model, action=\"generateContent\"):\n return f\"{GEMINI_BASE}/{model}:{action}?key={CFG['GEMINI_API_KEY']}\"\n\n\ndef call_gemini(b64data, mime, prompt, model=\"gemini-2.5-flash\"):\n payload = {\n \"contents\": [{\"parts\": [\n {\"text\": prompt},\n {\"inline_data\": {\"mime_type\": mime, \"data\": b64data}},\n ]}],\n }\n req = urllib.request.Request(\n _gemini_url(model),\n data=json.dumps(payload).encode(),\n headers={\"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=30)\n d = json.loads(resp.read())\n return d[\"candidates\"][0][\"content\"][\"parts\"][0][\"text\"]\n\n\ndef call_gemini_multi(frames, prompt, model=\"gemini-2.5-flash\"):\n parts = [{\"text\": prompt}]\n for frame in frames:\n if isinstance(frame, dict):\n parts.append({\"inline_data\": {\"mime_type\": frame.get(\"mime_type\", \"image/jpeg\"), \"data\": frame[\"data\"]}})\n elif isinstance(frame, tuple):\n data, mime = frame\n b64data = b64(data) if isinstance(data, bytes) else data\n parts.append({\"inline_data\": {\"mime_type\": mime, \"data\": b64data}})\n else:\n parts.append({\"inline_data\": {\"mime_type\": \"image/jpeg\", \"data\": frame}})\n payload = {\"contents\": [{\"parts\": parts}]}\n req = urllib.request.Request(\n _gemini_url(model),\n data=json.dumps(payload).encode(),\n headers={\"Content-Type\": \"application/json\"},\n )\n resp = urllib.request.urlopen(req, timeout=30)\n d = json.loads(resp.read())\n return d[\"candidates\"][0][\"content\"][\"parts\"][0][\"text\"]\n\n\n# ── ZAI (Zhipu AI) caller (OpenAI-compatible) ──────────────────────\nZAI_ENDPOINT = \"https://api.z.ai/api/paas/v4/chat/completions\"\n\n\ndef _zai_headers():\n return {\n \"Authorization\": f\"Bearer {CFG['ZAI_API_KEY']}\",\n \"Content-Type\": \"application/json\",\n }\n\n\ndef call_zai(b64data, mime, prompt, model=\"glm-4.5-flash\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": prompt},\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{mime};base64,{b64data}\"}},\n ]}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n ZAI_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_zai_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\ndef call_zai_multi(frames, prompt, model=\"glm-4.5-flash\"):\n payload = {\n \"model\": model,\n \"messages\": [{\"role\": \"user\", \"content\": build_multimodal_content(frames, prompt)}],\n \"max_tokens\": 2048,\n }\n req = urllib.request.Request(\n ZAI_ENDPOINT,\n data=json.dumps(payload).encode(),\n headers=_zai_headers(),\n )\n resp = urllib.request.urlopen(req, timeout=30)\n return json.loads(resp.read())[\"choices\"][0][\"message\"][\"content\"]\n\n\n# ── Provider routing ───────────────────────────────────────────────────\n\ndef get_providers_for_model(model):\n ml = model.lower()\n if \"/\" in model:\n prefix = model.split(\"/\", 1)[0].lower()\n stripped = model.split(\"/\", 1)[1]\n if prefix in (\"ollama\",):\n return [(\"ollama\", stripped)]\n if prefix in (\"lmstudio\", \"lm-studio\", \"openai-local\"):\n return [(\"lmstudio\", stripped)]\n if prefix in (\"openrouter\", \"or\"):\n return _filter_providers([(\"openrouter\", stripped)])\n if prefix in (\"openai\",):\n return _filter_providers([(\"openai\", stripped)])\n if prefix in (\"anthropic\", \"claude\"):\n return _filter_providers([(\"anthropic\", stripped)])\n if prefix in (\"together\", \"togetherai\"):\n return _filter_providers([(\"together\", stripped)])\n if prefix in (\"deepinfra\",):\n return _filter_providers([(\"deepinfra\", stripped)])\n if prefix in (\"cohere\",):\n return _filter_providers([(\"cohere\", stripped)])\n if prefix in (\"xai\", \"grok\"):\n return _filter_providers([(\"xai\", stripped)])\n if prefix in (\"gemini\", \"google\"):\n return _filter_providers([(\"gemini\", stripped)])\n if prefix in (\"cloudflare\", \"cf\", \"@cf\"):\n return _filter_providers([(\"cloudflare\", model)])\n if prefix in (\"azure\", \"azureai\"):\n deployment = model.split(\"/\", 1)[1]\n return _filter_providers([(\"azureai\", deployment)])\n if prefix in (\"groq\",):\n return _filter_providers([(\"groq\", stripped)])\n if prefix == \"hf\":\n return _filter_providers([(\"hf\", stripped)])\n if prefix == \"huggingface\":\n return _filter_providers([(\"hf\", stripped)])\n if prefix == \"mistral\":\n return _filter_providers([(\"mistral\", stripped)])\n if prefix == \"zai\":\n return _filter_providers([(\"zai\", stripped)])\n if prefix == \"fireworks\":\n return _filter_providers([(\"fireworks\", stripped)])\n if ml in (\"mistral\", \"pixtral-large-latest\", \"pixtral\"):\n return _filter_providers([(\"mistral\", ml)])\n if ml in (\"zai\", \"glm-4.5-flash\", \"glm-5\", \"glm-5.1\"):\n return _filter_providers([(\"zai\", ml)])\n return []\n\n\ndef _filter_providers(candidates):\n PROVIDER_KEY_MAP = {\n \"ollama\": None,\n \"lmstudio\": None,\n \"openrouter\": \"OPENROUTER_API_KEY\",\n \"openai\": \"OPENAI_API_KEY\",\n \"anthropic\": \"ANTHROPIC_API_KEY\",\n \"together\": \"TOGETHER_API_KEY\",\n \"deepinfra\": \"DEEPINFRA_API_KEY\",\n \"cohere\": \"COHERE_API_KEY\",\n \"xai\": \"XAI_API_KEY\",\n \"gemini\": \"GEMINI_API_KEY\",\n \"cloudflare\": \"CLOUDFLARE_API_KEY\",\n \"azureai\": \"AZUREAI_API_KEY\",\n \"groq\": \"GROQ_API_KEY\",\n \"hf\": \"HF_TOKEN\",\n \"mistral\": \"MISTRAL_API_KEY\",\n \"fireworks\": \"FIREWORKS_API_KEY\",\n \"zai\": \"ZAI_API_KEY\",\n }\n seen = set()\n result = []\n for prov, m in candidates:\n if prov in seen:\n continue\n seen.add(prov)\n if prov not in PROVIDER_KEY_MAP:\n continue\n key_name = PROVIDER_KEY_MAP.get(prov)\n if key_name is None:\n result.append((prov, m))\n elif key_name and CFG and CFG.get(key_name):\n result.append((prov, m))\n return result\n\n\n# ── Total-timeout wrapper ──────────────────────────────────────────────\n\ndef _call_with_timeout(fn, timeout_sec=12):\n pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)\n fut = pool.submit(fn)\n try:\n return fut.result(timeout=timeout_sec)\n except concurrent.futures.TimeoutError:\n raise TimeoutError(f\"Backend timed out after {timeout_sec}s\")\n finally:\n pool.shutdown(wait=False)\n\n\n# ── Strategy builder ──────────────────────────────────────────────────────\n\ndef _build_strategies(kind, *args, prompt=\"\"):\n if kind == \"vid\":\n frames = args[0]\n s = [\n # Google Gemini — tried first (fastest, most reliable)\n (\"\\u2606 Gemini 2.5 Flash\", lambda: call_gemini_multi(frames, prompt, \"gemini-2.5-flash\")),\n (\"\\u2606 Gemini 3 Flash Preview\", lambda: call_gemini_multi(frames, prompt, \"gemini-3-flash-preview\")),\n (\"\\u2606 Gemini 2.0 Flash\", lambda: call_gemini_multi(frames, prompt, \"gemini-2.0-flash\")),\n (\"\\u2606 Gemini 2.0 Flash Lite\", lambda: call_gemini_multi(frames, prompt, \"gemini-2.0-flash-lite\")),\n (\"\\u2606 Gemini 2.5 Pro\", lambda: call_gemini_multi(frames, prompt, \"gemini-2.5-pro\")),\n (\"\\u2606 Gemini 3 Pro Preview\", lambda: call_gemini_multi(frames, prompt, \"gemini-3-pro-preview\")),\n # OpenAI / Anthropic\n (\"\\u2606 OpenAI gpt-4o-mini\", lambda: call_openai_multi(frames, prompt, \"gpt-4o-mini\")),\n (\"\\u2606 Anthropic Claude Sonnet\", lambda: call_anthropic_multi(frames, prompt, \"claude-sonnet-4-5\")),\n # OpenAI-compatible vision providers\n (\"\\u2606 Together Kimi-K2.5\", lambda: call_together_multi(frames, prompt, \"moonshotai/Kimi-K2.5\")),\n (\"\\u2606 DeepInfra Qwen2.5-VL-72B\", lambda: call_deepinfra_multi(frames, prompt, \"Qwen/Qwen2.5-VL-72B-Instruct\")),\n (\"\\u2606 Cohere Command A Vision\", lambda: call_cohere_multi(frames, prompt, \"command-a-vision-07-2025\")),\n (\"\\u2606 xAI Grok\", lambda: call_xai_multi(frames, prompt, \"grok-4.3\")),\n # Azure AI Foundry\n (\"\\u2606 Azure DeepSeek-V4-Pro\", lambda: call_azureai_multi(frames, prompt, \"DeepSeek-V4-Pro\")),\n (\"\\u2606 Azure gpt-4.1\", lambda: call_azureai_multi(frames, prompt, \"gpt-4.1\")),\n (\"\\u2606 Azure gpt-4.1-mini\", lambda: call_azureai_multi(frames, prompt, \"gpt-4.1-mini\")),\n (\"\\u2606 Azure gpt-4.1-nano\", lambda: call_azureai_multi(frames, prompt, \"gpt-4.1-nano\")),\n (\"\\u2606 Azure gpt-4o\", lambda: call_azureai_multi(frames, prompt, \"gpt-4o\")),\n (\"\\u2606 Azure gpt-4o-mini\", lambda: call_azureai_multi(frames, prompt, \"gpt-4o-mini\")),\n (\"\\u2606 Azure gpt-5.1\", lambda: call_azureai_multi(frames, prompt, \"gpt-5.1\")),\n (\"\\u2606 Azure gpt-5.4\", lambda: call_azureai_multi(frames, prompt, \"gpt-5.4\")),\n (\"\\u2606 Azure gpt-5.4-mini\", lambda: call_azureai_multi(frames, prompt, \"gpt-5.4-mini\")),\n (\"\\u2606 Azure gpt-5.4-nano\", lambda: call_azureai_multi(frames, prompt, \"gpt-5.4-nano\")),\n (\"\\u2606 Azure Kimi-K2.6\", lambda: call_azureai_multi(frames, prompt, \"Kimi-K2.6\")),\n (\"\\u2606 Azure Phi-4 multimodal\", lambda: call_azureai_multi(frames, prompt, \"Phi-4-multimodal-instruct\")),\n # Groq\n (\"\\u2606 Groq Llama 4 Scout 17B\", lambda: call_groq_multi(frames, prompt, \"meta-llama/llama-4-scout-17b-16e-instruct\")),\n # HuggingFace\n (\"\\u2606 HF Qwen3-VL-8B\", lambda: call_hf_multi(frames, prompt, \"Qwen/Qwen3-VL-8B-Instruct\")),\n # Mistral AI\n (\"\\u2606 Mistral pixtral-large\", lambda: call_mistral_multi(frames, prompt, \"pixtral-large-latest\")),\n # Fireworks AI\n (\"\\u2606 Fireworks Llama 3.2 90B Vision\", lambda: call_fireworks_multi(frames, prompt, \"accounts/fireworks/models/llama-v3p2-90b-vision-instruct\")),\n # Zhipu AI\n (\"\\u2606 ZAI Glm-4.5-Flash\", lambda: call_zai_multi(frames, prompt, \"glm-4.5-flash\")),\n ]\n else:\n img_b64, mime = args\n s = [\n # Google Gemini — tried first (fastest, most reliable)\n (\"\\u2606 Gemini 2.5 Flash\", lambda: call_gemini(img_b64, mime, prompt, \"gemini-2.5-flash\")),\n (\"\\u2606 Gemini 3 Flash Preview\", lambda: call_gemini(img_b64, mime, prompt, \"gemini-3-flash-preview\")),\n (\"\\u2606 Gemini 2.0 Flash\", lambda: call_gemini(img_b64, mime, prompt, \"gemini-2.0-flash\")),\n (\"\\u2606 Gemini 2.0 Flash Lite\", lambda: call_gemini(img_b64, mime, prompt, \"gemini-2.0-flash-lite\")),\n (\"\\u2606 Gemini 2.5 Pro\", lambda: call_gemini(img_b64, mime, prompt, \"gemini-2.5-pro\")),\n (\"\\u2606 Gemini 3 Pro Preview\", lambda: call_gemini(img_b64, mime, prompt, \"gemini-3-pro-preview\")),\n # OpenAI / Anthropic\n (\"\\u2606 OpenAI gpt-4o-mini\", lambda: call_openai(img_b64, mime, prompt, \"gpt-4o-mini\")),\n (\"\\u2606 Anthropic Claude Sonnet\", lambda: call_anthropic(img_b64, mime, prompt, \"claude-sonnet-4-5\")),\n # OpenAI-compatible vision providers\n (\"\\u2606 Together Kimi-K2.5\", lambda: call_together(img_b64, mime, prompt, \"moonshotai/Kimi-K2.5\")),\n (\"\\u2606 DeepInfra Qwen2.5-VL-72B\", lambda: call_deepinfra(img_b64, mime, prompt, \"Qwen/Qwen2.5-VL-72B-Instruct\")),\n (\"\\u2606 Cohere Command A Vision\", lambda: call_cohere(img_b64, mime, prompt, \"command-a-vision-07-2025\")),\n (\"\\u2606 xAI Grok\", lambda: call_xai(img_b64, mime, prompt, \"grok-4.3\")),\n # Azure AI Foundry\n (\"\\u2606 Azure DeepSeek-V4-Pro\", lambda: call_azureai(img_b64, mime, prompt, \"DeepSeek-V4-Pro\")),\n (\"\\u2606 Azure gpt-4.1\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-4.1\")),\n (\"\\u2606 Azure gpt-4.1-mini\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-4.1-mini\")),\n (\"\\u2606 Azure gpt-4.1-nano\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-4.1-nano\")),\n (\"\\u2606 Azure gpt-4o\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-4o\")),\n (\"\\u2606 Azure gpt-4o-mini\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-4o-mini\")),\n (\"\\u2606 Azure gpt-5.1\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-5.1\")),\n (\"\\u2606 Azure gpt-5.4\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-5.4\")),\n (\"\\u2606 Azure gpt-5.4-mini\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-5.4-mini\")),\n (\"\\u2606 Azure gpt-5.4-nano\", lambda: call_azureai(img_b64, mime, prompt, \"gpt-5.4-nano\")),\n (\"\\u2606 Azure Kimi-K2.6\", lambda: call_azureai(img_b64, mime, prompt, \"Kimi-K2.6\")),\n (\"\\u2606 Azure Phi-4 multimodal\", lambda: call_azureai(img_b64, mime, prompt, \"Phi-4-multimodal-instruct\")),\n # Groq\n (\"\\u2606 Groq Llama 4 Scout 17B\", lambda: call_groq(img_b64, mime, prompt, \"meta-llama/llama-4-scout-17b-16e-instruct\")),\n # HuggingFace\n (\"\\u2606 HF Qwen3-VL-8B\", lambda: call_hf_inference(img_b64, mime, prompt, \"Qwen/Qwen3-VL-8B-Instruct\")),\n # Mistral AI\n (\"\\u2606 Mistral pixtral-large\", lambda: call_mistral(img_b64, mime, prompt, \"pixtral-large-latest\")),\n # Fireworks AI\n (\"\\u2606 Fireworks Llama 3.2 90B Vision\", lambda: call_fireworks(img_b64, mime, prompt, \"accounts/fireworks/models/llama-v3p2-90b-vision-instruct\")),\n # Zhipu AI\n (\"\\u2606 ZAI Glm-4.5-Flash\", lambda: call_zai(img_b64, mime, prompt, \"glm-4.5-flash\")),\n ]\n return s\n\n\ndef _insert_model_strategies(strategies, model, kind, *args, prompt=\"\"):\n dispatch = {\n \"ollama\": (call_ollama, call_ollama_multi),\n \"lmstudio\": (call_lmstudio, call_lmstudio_multi),\n \"openrouter\": (call_openrouter, call_openrouter_multi),\n \"openai\": (call_openai, call_openai_multi),\n \"anthropic\": (call_anthropic, call_anthropic_multi),\n \"together\": (call_together, call_together_multi),\n \"deepinfra\": (call_deepinfra, call_deepinfra_multi),\n \"cohere\": (call_cohere, call_cohere_multi),\n \"xai\": (call_xai, call_xai_multi),\n \"cloudflare\": (call_cloudflare, call_cloudflare_multi),\n \"azureai\": (call_azureai, call_azureai_multi),\n \"groq\": (call_groq, call_groq_multi),\n \"hf\": (call_hf_inference, call_hf_multi),\n \"mistral\": (call_mistral, call_mistral_multi),\n \"fireworks\": (call_fireworks, call_fireworks_multi),\n \"gemini\": (call_gemini, call_gemini_multi),\n \"zai\": (call_zai, call_zai_multi),\n }\n is_vid = kind == \"vid\"\n for prov, native_model in reversed(get_providers_for_model(model)):\n pair = dispatch.get(prov)\n if not pair:\n continue\n fn_img, fn_vid = pair\n fn = fn_vid if is_vid else fn_img\n if is_vid:\n strategies.insert(0, (\n f\"\\u2605 {prov.title()}: {model}\",\n lambda m=native_model, f=fn: f(args[0], prompt, m),\n ))\n else:\n strategies.insert(0, (\n f\"\\u2605 {prov.title()}: {model}\",\n lambda m=native_model, f=fn: f(args[0], args[1], prompt, m),\n ))\n\n\n# ── Public API ──────────────────────────────────────────────────────────\n\ndef analyze(file_path, prompt=\"\", model=None):\n if os.path.isdir(file_path):\n raise FileNotFoundError(\n f\"Path is a directory, not a file: {file_path}\\n\"\n f\" Pass the full path to an image or video file.\"\n )\n\n if not os.path.isfile(file_path):\n print(\"SEARCH: Locating file...\", file=sys.stderr, flush=True)\n found = find_file(file_path, max_results=1)\n if found:\n file_path = found[0]\n print(f\"SEARCH: Found -> {file_path}\", file=sys.stderr, flush=True)\n else:\n print(\"SEARCH: Not found\", file=sys.stderr, flush=True)\n raise FileNotFoundError(\n f\"File not found: {file_path}\\n\"\n f\" Tried: Desktop, Downloads, Pictures, Documents, CWD, and user profile.\\n\"\n f\" Pass the full absolute path or make sure the file is on Desktop/Downloads/Pictures.\"\n )\n else:\n print(f\"SEARCH: File exists at {file_path}\", file=sys.stderr, flush=True)\n\n filename = os.path.basename(file_path)\n vid = is_video(file_path)\n\n global CFG\n CFG = load_config()\n start_background_capability_refresh(reason=\"analyze_start\")\n _print_available_keys()\n\n model = model or CFG.get(\"DEFAULT_MODEL\", \"\") or None\n\n if not prompt:\n if vid:\n prompt = (\n \"Describe this video naturally — what's happening, what you see changing \"\n \"frame to frame. Cover the layout, any visible text or UI elements, \"\n \"colors, and scene transitions. Be thorough but conversational.\"\n )\n else:\n prompt = (\n \"What do you see in this image? Describe it naturally — cover the main \"\n \"subject, layout, any visible text, colors, and visual style. \"\n \"If it's a UI or design screenshot, note key elements, spacing, and how \"\n \"things are arranged. Be thorough but conversational.\"\n )\n\n if vid:\n frames = extract_video_frames(file_path, max_frames=8)\n strategies = _build_strategies(\"vid\", frames, prompt=prompt)\n else:\n data, mime = resize_image(file_path, 1024)\n img_b64 = b64(data)\n strategies = _build_strategies(\"img\", img_b64, mime, prompt=prompt)\n\n if model:\n _insert_model_strategies(strategies, model, \"vid\" if vid else \"img\",\n *(frames if vid else (img_b64, mime)), prompt=prompt)\n\n before = len(strategies)\n strategies = [(n, f) for n, f in strategies if _has_key(n)]\n skipped = before - len(strategies)\n if skipped:\n print(f\"KEYS: Skipped {skipped}/{before} backends (missing API key)\", file=sys.stderr, flush=True)\n\n backend_memory = _load_backend_memory()\n strategies, memory_skipped = _filter_strategies_by_memory(strategies, backend_memory)\n if memory_skipped:\n print(\n f\"MEMORY: Skipped {len(memory_skipped)} backends in 24h limit cooldown\",\n file=sys.stderr,\n flush=True,\n )\n for name, retry_in in memory_skipped[:5]:\n hours = max(1, int((retry_in + 3599) / 3600))\n print(f\" {name}: retry in ~{hours}h\", file=sys.stderr, flush=True)\n if len(memory_skipped) > 5:\n print(f\" ... {len(memory_skipped) - 5} more\", file=sys.stderr, flush=True)\n print(f\"KEYS: {len(strategies)} backends available\", file=sys.stderr, flush=True)\n\n if not strategies:\n if memory_skipped:\n retry_in = min(seconds for _, seconds in memory_skipped)\n hours = max(1, int((retry_in + 3599) / 3600))\n raise RuntimeError(\n f\"All configured vision backends are in 24h limit cooldown. \"\n f\"Next retry in ~{hours}h.\"\n )\n raise RuntimeError(\"No backends available — configure at least one API key (python setup.py)\")\n\n PER_CALL_TIMEOUT = 12\n TOTAL_TIMEOUT = 25\n FAST_TIMEOUT = 4\n FAST_SEQUENTIAL = min(2, len(strategies))\n warmup_memory = _has_unknown_backend_status(strategies, backend_memory)\n last_error = \"\"\n first_success = None\n\n if warmup_memory:\n print(\n \"MEMORY: Mapping backend health; unknown models will be remembered for future runs\",\n file=sys.stderr,\n flush=True,\n )\n\n for name, fn in strategies[:FAST_SEQUENTIAL]:\n try:\n text = _call_with_timeout(fn, FAST_TIMEOUT)\n if text and text.strip():\n _record_backend_success(backend_memory, name)\n print(f\" {name}: OK\", file=sys.stderr, flush=True)\n if not warmup_memory:\n return f\"[{filename}]\\n{text}\"\n if first_success is None:\n first_success = f\"[{filename}]\\n{text}\"\n except Exception as e:\n msg = str(e)\n if hasattr(e, \"code\"):\n msg = f\"HTTP {e.code}\"\n last_error = msg\n _record_backend_failure(backend_memory, name, e, msg)\n print(f\" {name}: FAILED ({msg})\", file=sys.stderr, flush=True)\n\n remaining = strategies[FAST_SEQUENTIAL:]\n if not remaining:\n if first_success is not None:\n return first_success\n raise RuntimeError(f\"All vision backends failed. Last error: {last_error}\")\n\n print(f\"FALLBACK: {len(remaining)} backends in parallel\", file=sys.stderr, flush=True)\n pool = concurrent.futures.ThreadPoolExecutor(max_workers=len(remaining))\n futs = {pool.submit(lambda f=fn, n=name: (n, _call_with_timeout(f, PER_CALL_TIMEOUT))): name for name, fn in remaining}\n try:\n for fut in concurrent.futures.as_completed(futs, timeout=TOTAL_TIMEOUT):\n name = futs[fut]\n try:\n text = fut.result()[1]\n if text and text.strip():\n _record_backend_success(backend_memory, name)\n print(f\" {name}: OK\", file=sys.stderr, flush=True)\n if not warmup_memory:\n return f\"[{filename}]\\n{text}\"\n if first_success is None:\n first_success = f\"[{filename}]\\n{text}\"\n except Exception as e:\n msg = str(e)\n if hasattr(e, \"code\"):\n msg = f\"HTTP {e.code}\"\n last_error = msg\n _record_backend_failure(backend_memory, name, e, msg)\n print(f\" {name}: FAILED ({msg})\", file=sys.stderr, flush=True)\n except concurrent.futures.TimeoutError:\n pass\n finally:\n pool.shutdown(wait=False, cancel_futures=True)\n\n if first_success is not None:\n return first_success\n\n raise RuntimeError(f\"All vision backends failed. Last error: {last_error}\")\n\n\n# ── CLI entry point ─────────────────────────────────────────────────────\n\ndef _list_models():\n global CFG\n CFG = load_config(require_keys=False)\n\n models = [\n # Azure AI Foundry\n (\"\\u2606 Azure DeepSeek-V4-Pro\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-4.1\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-4.1-mini\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-4.1-nano\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-4o\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-4o-mini\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-5.1\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-5.4\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-5.4-mini\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure gpt-5.4-nano\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure Kimi-K2.6\", \"AZUREAI_API_KEY\"),\n (\"\\u2606 Azure Phi-4 multimodal\", \"AZUREAI_API_KEY\"),\n # Groq\n (\"\\u2606 Groq Llama 4 Scout 17B\", \"GROQ_API_KEY\"),\n # HuggingFace\n (\"\\u2606 HF Qwen3-VL-8B\", \"HF_TOKEN\"),\n # Mistral AI\n (\"\\u2606 Mistral pixtral-large\", \"MISTRAL_API_KEY\"),\n # Fireworks AI\n (\"\\u2606 Fireworks Llama 3.2 90B Vision\", \"FIREWORKS_API_KEY\"),\n # Zhipu AI\n (\"\\u2606 ZAI Glm-4.5-Flash\", \"ZAI_API_KEY\"),\n # Google Gemini\n (\"\\u2606 Gemini 2.5 Flash\", \"GEMINI_API_KEY\"),\n (\"\\u2606 Gemini 3 Flash Preview\", \"GEMINI_API_KEY\"),\n (\"\\u2606 Gemini 2.0 Flash\", \"GEMINI_API_KEY\"),\n (\"\\u2606 Gemini 2.0 Flash Lite\", \"GEMINI_API_KEY\"),\n (\"\\u2606 Gemini 2.5 Pro\", \"GEMINI_API_KEY\"),\n (\"\\u2606 Gemini 3 Pro Preview\", \"GEMINI_API_KEY\"),\n # OpenRouter\n (\"\\u2606 OpenRouter configured vision model\", \"OPENROUTER_API_KEY\"),\n # First-party / gateway vision providers\n (\"\\u2606 OpenAI gpt-4o-mini\", \"OPENAI_API_KEY\"),\n (\"\\u2606 Anthropic Claude Sonnet\", \"ANTHROPIC_API_KEY\"),\n (\"\\u2606 Together Kimi-K2.5\", \"TOGETHER_API_KEY\"),\n (\"\\u2606 DeepInfra Qwen2.5-VL-72B\", \"DEEPINFRA_API_KEY\"),\n (\"\\u2606 Cohere Command A Vision\", \"COHERE_API_KEY\"),\n (\"\\u2606 xAI Grok\", \"XAI_API_KEY\"),\n ]\n\n native_default = CFG.get(\"DEFAULT_MODEL\", \"\") or \"\"\n if native_default and \"/\" in native_default:\n prefix, native_name = native_default.split(\"/\", 1)\n label = {\n \"ollama\": \"Ollama\",\n \"lmstudio\": \"LM Studio\",\n \"openai-local\": \"OpenAI Local\",\n \"openrouter\": \"OpenRouter\",\n \"gemini\": \"Gemini\",\n \"mistral\": \"Mistral\",\n \"groq\": \"Groq\",\n \"hf\": \"HuggingFace\",\n \"huggingface\": \"HuggingFace\",\n \"fireworks\": \"Fireworks\",\n \"zai\": \"ZAI\",\n \"azureai\": \"Azure AI\",\n \"openai\": \"OpenAI\",\n \"anthropic\": \"Anthropic\",\n \"claude\": \"Anthropic\",\n \"together\": \"Together\",\n \"deepinfra\": \"DeepInfra\",\n \"cohere\": \"Cohere\",\n \"xai\": \"xAI\",\n \"grok\": \"xAI\",\n }.get(prefix.lower(), prefix.replace(\"-\", \" \").title())\n models.insert(0, (f\"\\u2606 {label} {native_name}\", \"DEFAULT_MODEL\"))\n\n print(f\"{'Vision Backend':<45} {'Status':<12} Key\")\n print(\"-\" * 75)\n for name, key in models:\n has = bool(CFG.get(key))\n status = \"AVAILABLE\" if has else \"NO KEY\"\n status_sym = \"\\u2705\" if has else \"\\u274c\"\n print(f\" {name:<42} {status_sym} {status:<8} {key}\")\n\n print()\n print(\"Keys configured:\")\n for key in [\"DEFAULT_MODEL\", \"GEMINI_API_KEY\", \"OPENROUTER_API_KEY\", \"OPENAI_API_KEY\", \"ANTHROPIC_API_KEY\", \"TOGETHER_API_KEY\", \"DEEPINFRA_API_KEY\", \"COHERE_API_KEY\", \"XAI_API_KEY\", \"CLOUDFLARE_API_KEY\", \"AZUREAI_API_KEY\", \"AZUREAI_ENDPOINT\", \"MISTRAL_API_KEY\", \"GROQ_API_KEY\", \"HF_TOKEN\", \"FIREWORKS_API_KEY\", \"ZAI_API_KEY\"]:\n v = CFG.get(key, \"\")\n val = v[:20] + \"...\" if v and len(v) > 20 else (v or \"(not set)\")\n print(f\" {key:<25} {val}\")\n\n\ndef main():\n import argparse\n parser = argparse.ArgumentParser(\n description=\"Analyse images and videos using AI vision models.\",\n epilog=\"First run? python setup.py\",\n )\n parser.add_argument(\"file\", nargs=\"?\", help=\"Path to image or video file\")\n parser.add_argument(\"prompt\", nargs=\"*\", help=\"Optional prompt text\")\n parser.add_argument(\"--model\", \"-m\", help=\"Custom model name (auto-routes to best provider)\")\n parser.add_argument(\"--models\", \"-l\", action=\"store_true\", help=\"List available vision backends and exit\")\n parser.add_argument(\"--refresh-profile\", action=\"store_true\", help=\"Refresh provider free/paid capability profile now\")\n args = parser.parse_args()\n\n if args.refresh_profile:\n refresh_capability_profile(force=True, reason=\"cli\")\n memory = _load_backend_memory()\n profile = memory.get(\"capability_profile\", {})\n print(json.dumps(profile, indent=2, sort_keys=True))\n return\n\n if args.models:\n start_background_capability_refresh(reason=\"models_start\")\n _list_models()\n return\n\n if not args.file:\n parser.print_help()\n sys.exit(1)\n\n file_path = args.file\n prompt = \" \".join(args.prompt) if args.prompt else \"\"\n model = args.model\n\n try:\n result = analyze(file_path, prompt, model)\n print(result)\n except FileNotFoundError as e:\n print(str(e), file=sys.stderr)\n sys.exit(1)\n except RuntimeError as e:\n print(str(e), file=sys.stderr)\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}}