1 line
79 KiB
JSON
1 line
79 KiB
JSON
{"repo": "abhishekvulla/clicky-windows", "files": {"config.py": {"04f221f91d4750363d7032bcfd355b7dbc5c0e06": {"t": 1785674356.7880626, "n": 24441, "h": "11708f681fa5cdca", "c": "\"\"\"Clicky Windows configuration.\n\nLoads environment variables from .env (Phase 1 BYOK pattern) AND from\nthe OS keyring (Sprint 3 — Windows Credential Manager via the keyring\npackage, DPAPI per-user encryption). On launch with .env present, the\nkeys are auto-migrated to the keyring as a backup; user can then delete\n.env without losing the keys.\n\nSee DECISIONS.md for the rationale behind each default.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport os\nfrom pathlib import Path\n\nimport keyring\nfrom dotenv import load_dotenv\n\nload_dotenv()\n\n\n# ── Secrets resolution (env → keyring with one-shot migration) ──────────────\n\nKEYRING_SERVICE: str = \"clicky-windows\"\n\"\"\"Service name for keyring entries. Both Windows Credential Manager\nand macOS Keychain treat this as the namespace key. All Clicky API\nkeys live under this single service name; the ``name`` parameter is\nthe env-var name (ANTHROPIC_API_KEY, etc.).\"\"\"\n\n\ndef resolve_api_key(name: str) -> str | None:\n \"\"\"Resolve an API key by name, preferring env var then keyring.\n\n On env-var-present, ALSO write the value to keyring as a backup —\n this is the one-shot migration path from Phase 1's ``.env`` workflow\n to Phase 2's keyring storage. Subsequent launches with no .env will\n pick up the value from keyring transparently.\n\n Failures in keyring (locked vault, no backend, transient errors)\n are swallowed — the env-var path always works as a fallback. We\n never want a credential-store glitch to block app startup when the\n user has perfectly valid keys in their .env.\n\n Returns None if neither source has a value (caller shows the\n first-launch settings dialog).\n \"\"\"\n env_value = os.getenv(name)\n if env_value:\n try:\n keyring.set_password(KEYRING_SERVICE, name, env_value)\n except Exception:\n # Keyring backend unreachable; env value is still good.\n pass\n return env_value\n try:\n return keyring.get_password(KEYRING_SERVICE, name)\n except Exception:\n return None\n\n\ndef resolve_setting(name: str, default: str) -> str:\n \"\"\"Resolve a non-secret setting by name with env→keyring→default fallback.\n\n Sibling to ``resolve_api_key`` for config knobs (TTS_PROVIDER,\n LLM_PROVIDER, STT_PROVIDER, etc.) that need keyring persistence so\n bundled-EXE startup doesn't silently fall back to defaults when the\n user's `.env` doesn't load (cwd is install dir, not repo root — see\n DECISIONS.md 2026-05-05 Sprint 3.6).\n\n Differs from resolve_api_key in that it always returns a string —\n callers pass the right default for the setting (e.g. \"cartesia\" for\n TTS_PROVIDER) rather than handling None.\n\n Failures in keyring (locked vault, no backend) are swallowed in both\n directions: env path always returns successfully even if keyring write\n fails; keyring read errors fall through to the default.\n \"\"\"\n env_value = os.getenv(name)\n if env_value:\n try:\n keyring.set_password(KEYRING_SERVICE, name, env_value)\n except Exception:\n pass\n return env_value\n try:\n stored = keyring.get_password(KEYRING_SERVICE, name)\n except Exception:\n stored = None\n return stored if stored else default\n\n\n# ── API keys ─────────────────────────────────────────────────────────────────\n\nANTHROPIC_API_KEY: str | None = resolve_api_key(\"ANTHROPIC_API_KEY\")\n\"\"\"Required. Plain vision streaming via messages.stream(). Sonnet 4.6 default.\"\"\"\n\nASSEMBLYAI_API_KEY: str | None = resolve_api_key(\"ASSEMBLYAI_API_KEY\")\n\"\"\"Required for Phase 1. Streaming STT via AssemblyAI u3-rt-pro WebSocket +\nForceEndpoint for ~150ms P50 PTT finalization. $50 free credit from\nhttps://www.assemblyai.com/dashboard/signup, no credit card required.\"\"\"\n\nCARTESIA_API_KEY: str | None = resolve_api_key(\"CARTESIA_API_KEY\")\n\"\"\"Required for Phase 1. Streaming TTS via Cartesia Sonic-3 WebSocket with\n~150-250ms TTFB + expressive \"buddy\" voice. 20k free credits/month from\nhttps://play.cartesia.ai/sign-in, no credit card required.\"\"\"\n\nOPENAI_API_KEY: str | None = resolve_api_key(\"OPENAI_API_KEY\")\n\"\"\"Optional (v0.3.0). OpenAI native API key (sk-...) for the OpenAI LLM\nprovider — GPT-4o vision in the normal pipeline, and GPT-Realtime\nspeech-to-speech as a separate path. Selected via LLM_PROVIDER='openai'\n(GPT-4o) or 'openai-realtime' (speech-to-speech). Distinct from the\nOpenRouter sk-or- key stored in ANTHROPIC_API_KEY — this is a direct\nOpenAI key for api.openai.com. Get one at\nhttps://platform.openai.com/api-keys.\"\"\"\n\n\n# ── OpenRouter dual-SDK routing (BYOK, model-agnostic) ──────────────────────\n\nOPENROUTER_BASE_URL: str = \"https://openrouter.ai/api/v1\"\n\"\"\"OpenRouter's OpenAI-compatible endpoint for Gemini / Grok / Llama / etc.\n\nThe existing ANTHROPIC_BASE_URL env var (read natively by the Anthropic SDK)\npoints at 'https://openrouter.ai/api' for Claude models. This constant is the\nsibling endpoint for the OpenAI SDK used by GeminiClient (ai.py). Same API\nkey (ANTHROPIC_API_KEY from .env — which is actually the OpenRouter\nsk-or-v1-... key when ANTHROPIC_BASE_URL is set to OpenRouter).\n\nSee DECISIONS.md 2026-04-19 'Gemini 3 Flash via OpenRouter' for the full\ndual-SDK routing rationale.\"\"\"\n\n\n# ── LLM model ID (routed by prefix via ai.create_ai_client) ─────────────────\n\nMODEL_ID: str = os.getenv(\"MODEL_ID\", \"anthropic/claude-sonnet-4-6\")\n\"\"\"OpenRouter-style model ID. Prefix routes to the right SDK via\nai.create_ai_client():\n 'anthropic/...' → AnthropicClient (via anthropic SDK, OpenRouter\n Anthropic-compat endpoint)\n 'google/...' → GeminiClient (via openai SDK, OpenRouter OpenAI-compat\n endpoint)\n\nPhase 1.5 default is 'google/gemini-3-flash-preview' for ~50% latency\nreduction on vision tasks. Set in .env to override.\n\nSee DECISIONS.md 2026-04-19 for the model swap rationale + latency\nbenchmarks.\"\"\"\n\n\n# ── Screen capture ───────────────────────────────────────────────────────────\n\nCANDIDATE_RESOLUTIONS: list[tuple[int, int]] = [\n (1024, 768), # 4:3 = 1.333 (legacy displays)\n (1280, 800), # 16:10 = 1.600 (most laptops)\n (1366, 768), # ~16:9 = 1.779 (external monitors, ultrawide fallback)\n]\n\"\"\"Anthropic-recommended screenshot resolutions. capture.py picks the\nclosest-aspect-ratio pair to the actual monitor to avoid distortion. Mirrors\nClicky's CompanionScreenCaptureUtility.swift (max dimension 1280).\"\"\"\n\n\n# ── Hotkey ───────────────────────────────────────────────────────────────────\n\nHOTKEY: str = os.getenv(\"HOTKEY\", \"ctrl+alt+space\")\n\"\"\"Default push-to-talk hotkey. Ctrl+Alt+Space because:\n\n 1. Alt+Space alone conflicts with the Windows window menu + Copilot\n (Microsoft reassigned it in Windows 11, late 2024). Making it work\n cleanly needs Win32 RegisterHotKey + GetAsyncKeyState polling for\n release detection -- 8-12h of fragile ctypes code, deferred to\n Phase 1.5 as a drop-in subclass.\n 2. Ctrl+Shift+Space was our earlier pivot target but conflicts with\n Microsoft Excel + Google Sheets \"Select entire worksheet\" binding.\n Because our pynput listener uses suppress=False (observe-only),\n the spreadsheet underneath ALSO receives the keypress and wipes\n the user's selection every time they invoke Clicky -- a showstopper\n for the Excel-learning demo narrative.\n 3. Fn+Space is firmware-level (handled by the keyboard EC below the\n OS) and invisible to WH_KEYBOARD_LL + pynput. Non-portable even\n where it happens to work. AutoHotkey docs: \"the Fn key does not\n (as a general rule) generate any scan code that can be used.\"\n 4. Ctrl+Alt+Space has no known code-level conflicts (Excel, Sheets,\n Windows menu, Copilot, VS Code all clear). Three-finger but all on\n the left side of the keyboard for one-handed ergonomics. suppress=\n False observe-only model carries over unchanged.\n\n KNOWN SETUP REQUIREMENT: if you have Claude Desktop for Windows\n installed, disable its Ctrl+Alt+Space binding in Claude Desktop\n Settings > Keyboard Shortcuts (same pattern Raycast / Flow Launcher\n users follow for Alt+Space). Clicky's listener is observe-only so\n both apps receive the keypress otherwise, and Claude Desktop's\n quick-access prompt will pop every time you invoke Clicky. See\n DECISIONS.md 2026-04-12 (evening) entry for the full rationale +\n the Phase 1.5 Win32 RegisterHotKey solution that eliminates the\n conflict at the OS level.\n\nNEVER ctrl+space (VS Code IntelliSense conflict -- still rejected).\n\nSee DECISIONS.md 2026-04-12 (evening) entry \"Ctrl+Alt+Space replaces\nCtrl+Shift+Space\" for the full pivot story + research-backed Fn+Space\nrejection. Phase 1.5 Win32 RegisterHotKey subclass will restore Alt+Space\nergonomics via the abstract PushToTalkHotkey interface.\"\"\"\n\n\n# ── STT (AssemblyAI u3-rt-pro streaming) ─────────────────────────────────────\n\nASSEMBLYAI_SPEECH_MODEL: str = \"u3-rt-pro\"\n\"\"\"AssemblyAI Universal-3 realtime-pro. Matches Clicky's Swift source\n(leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift:447-451).\n~150ms P50 finalization after ForceEndpoint message on hotkey release.\"\"\"\n\nASSEMBLYAI_STREAMING_URL: str = \"wss://streaming.assemblyai.com/v3/ws\"\n\"\"\"AssemblyAI streaming WebSocket endpoint. Query params are set via SDK.\"\"\"\n\nAUDIO_SAMPLE_RATE: int = 16_000\n\"\"\"PCM16 mono at 16kHz. Matches AssemblyAI u3-rt-pro's required sample rate +\nClicky's audio pipeline + the canonical input shape for every major\nstreaming STT provider.\"\"\"\n\nAUDIO_CHUNK_FRAMES: int = 1024\n\"\"\"sounddevice RawInputStream blocksize. Matches Clicky's\nAVAudioEngine.installTap(onBus:0, bufferSize:1024) exactly so the streaming\nWebSocket payload shape is identical for Phase 2 provider swaps.\"\"\"\n\n# ── Audio level (RMS) filter — drives the waveform widget ──────────────────\n\nAUDIO_POWER_BOOST: float = 10.2\n\"\"\"Multiplier applied to per-chunk RMS before clamping to [0, 1]. Tuned to\nmake normal speech register ~0.4-0.8 on the waveform. Matches Farza's\nleanring-buddy/BuddyDictationManager.swift:687-721 verbatim.\"\"\"\n\nAUDIO_POWER_DECAY: float = 0.72\n\"\"\"Exponential decay floor between chunks: smoothed = max(raw, old * 0.72).\nPrevents the UI waveform from jumping DOWN sharply at natural speech pauses —\nmakes the meter feel responsive to loud sounds but stable at quiet ones.\nMatches Farza's implementation.\"\"\"\n\n\n# ── TTS (Cartesia Sonic-3 WebSocket streaming) ──────────────────────────────\n\nCARTESIA_MODEL_ID: str = \"sonic-3\"\n\"\"\"Cartesia's state-space-model-based TTS. ~90ms model-internal TTFB,\n150-250ms real-world through the WebSocket stream + sounddevice playback.\nMost expressive 'buddy' voice quality in the cloud TTS field as of April 2026.\nSee DECISIONS.md 'Priority inversion' for the research.\"\"\"\n\nCARTESIA_VOICE_ID: str = os.getenv(\n \"CARTESIA_VOICE_ID\",\n \"f786b574-daa5-4673-aa0c-cbe3e8534c02\", # \"Katie - Friendly Fixer\" — Cartesia-recommended for voice agents\n)\n\"\"\"Cartesia voice ID for Sonic-3. Default is \"Brooke - Big Sister\" — a confident\nadult female voice described as \"for conversational use cases\" in Cartesia's\nvoice catalog. The \"big sister\" framing matches our \"buddy next to you\" UX.\n\nSwap via .env CARTESIA_VOICE_ID=... if Brooke doesn't land for the demo.\nOther strong feminine candidates from the Cartesia catalog:\n - Cathy - Coworker (e8e5fffb-252c-436d-b842-8879b84445b6) — \"nice young adult female for casual conversations\"\n - Skylar - Friendly Guide (db6b0ed5-d5d3-463d-ae85-518a07d3c2b4) — \"approachable American female\"\n - Lauren - Lively Narrator (a33f7a4c-100f-41cf-a1fd-5822e8fc253f) — \"expressive female, narration, storytelling\" (most dramatic/emotive)\n - Katie - Friendly Fixer (f786b574-daa5-4673-aa0c-cbe3e8534c02) — \"enunciating young adult female, conversational support\"\nThe previous default (a0e99841...) was a hallucinated UUID I made up without\nverifying against Cartesia's catalog — sorry. User reported it as \"kinda robotic\"\nwhich is probably because Cartesia fell back to a default voice.\"\"\"\n\nCARTESIA_OUTPUT_SAMPLE_RATE: int = 44_100\n\"\"\"Cartesia output stream sample rate. 44.1 kHz PCM float32 via sounddevice\nOutputStream. Cartesia supports 22.05k / 44.1k / 48k — 44.1k is the most\nnatural for buddy voice without oversampling cost.\"\"\"\n\n\n# ── Provider selection (which subclass app.py constructs at startup) ────────\n\nLLM_PROVIDER: str = resolve_setting(\"LLM_PROVIDER\", default=\"anthropic\")\n\"\"\"Which AIClient subclass to construct. Sprint 4 ships only \"anthropic\"\nin the dropdown; GeminiClient infrastructure stays in ai.py for opt-in\nvia env override (MODEL_ID=google/...) but is not user-selectable in the\nsettings dialog. See DECISIONS.md 2026-04-19 (late-evening) for the\nempirical A/B that rejected Gemini on coordinate accuracy.\"\"\"\n\nSTT_PROVIDER: str = resolve_setting(\"STT_PROVIDER\", default=\"assemblyai\")\n\"\"\"Which STT subclass to construct. Sprint 4 ships only \"assemblyai\".\nDeepgram is parked for post-launch.\"\"\"\n\nTTS_PROVIDER: str = resolve_setting(\"TTS_PROVIDER\", default=\"cartesia\")\n\"\"\"Which TTS subclass to construct. Sprint 4 ships \"cartesia\" (default)\nand \"elevenlabs\" (opt-in). User switches via Settings dialog dropdown.\"\"\"\n\nLLM_PROVIDER: str = resolve_setting(\"LLM_PROVIDER\", default=\"anthropic\")\n\"\"\"Which LLM subclass to construct (v0.2.0). Settings dialog dropdown\nwrites this. Values: 'anthropic' (default, cloud) or 'ollama' (local).\n\nWhen 'ollama' is selected, app.py builds an OllamaClient pointed at\nOLLAMA_HOST with OLLAMA_MODEL_VISION as the model — completely bypassing\nANTHROPIC_API_KEY (which may be empty). Without this branch, the\nSettings dropdown was cosmetic in v0.2.0 (caught by codex adversarial\nreview 2026-06-05): app would silently fall back to AnthropicClient\nwith whatever MODEL_ID env var said, ignoring the user's choice.\n\nTo override via env var (advanced): set MODEL_ID=ollama/llama3.2-vision\ndirectly — that takes precedence over LLM_PROVIDER because the create_ai_client\nfactory dispatches on MODEL_ID prefix first. LLM_PROVIDER is the\nGUI-friendly way for users who don't edit .env.\"\"\"\n\nANNOTATION_MODE: str = resolve_setting(\"ANNOTATION_MODE\", default=\"off\")\n\"\"\"Draw-on-screen teaching mode (v0.3.0, hackathon). When 'on', the vision\nmodel is given the annotation system prompt and emits\n[ARROW]/[CIRCLE]/[UNDERLINE]/[LABEL] tags that the overlay renders as shapes\n(in ADDITION to the [POINT] cursor). When 'off' (default) Clicky behaves\nexactly as before — nothing is overridden. Accuracy comes from the model\n(Claude is natively precise; GPT-4o/Ollama selectable). Resolved ONCE here at\nimport (env→keyring→default); app.py reads this cached constant per interaction\nrather than calling resolve_setting on the hot path, so there is no\nper-interaction keyring read/write latency. Set it in .env and restart to\ntoggle.\"\"\"\n\n\n# ── ElevenLabs TTS (opt-in alternative to Cartesia) ─────────────────────────\n\nELEVENLABS_API_KEY: str | None = resolve_api_key(\"ELEVENLABS_API_KEY\")\n\"\"\"Optional. Required only when TTS_PROVIDER='elevenlabs'. 10k chars/month\nfree tier at https://elevenlabs.io/app/sign-up — no credit card.\"\"\"\n\nELEVENLABS_MODEL_ID: str = os.getenv(\"ELEVENLABS_MODEL_ID\", \"eleven_flash_v2_5\")\n\"\"\"ElevenLabs Flash v2.5 — ~75ms model TTFB. ElevenLabs officially\nrecommends Flash over Turbo v2.5 for low-latency voice agents.\nVerified 2026-05-06 against ElevenLabs Python SDK 2.45.0 (\n``client.text_to_speech.stream`` accepts ``model_id=\"eleven_flash_v2_5\"``).\n\"\"\"\n\nELEVENLABS_VOICE_ID: str = os.getenv(\n \"ELEVENLABS_VOICE_ID\",\n \"21m00Tcm4TlvDq8ikWAM\", # Rachel — American female, conversational\n)\n\"\"\"ElevenLabs voice ID for the buddy persona. Default Rachel matches\nCartesia \"Brooke - Big Sister\" warmth (conversational adult female).\nVerified 2026-05-06 against ElevenLabs voice catalog\n(https://elevenlabs.io/app/voice-library) — Rachel's official voice ID\nis ``21m00Tcm4TlvDq8ikWAM``. If swapping to a different voice via env\noverride, copy the ID from the voice library page (NOT the URL slug).\"\"\"\n\nELEVENLABS_OUTPUT_SAMPLE_RATE: int = int(\n os.getenv(\"ELEVENLABS_OUTPUT_SAMPLE_RATE\", \"22050\")\n)\n\"\"\"ElevenLabs PCM sample rate. Defaulted to 22050 because 44.1kHz PCM\nrequires Pro tier. ElevenLabs PCM is int16 (NOT float32 like Cartesia),\nso playback path converts inline: np.frombuffer(chunk, np.int16).astype(\nnp.float32) / 32768.0.\"\"\"\n\n\n# ── Ollama (local LLM via Ollama server) — added v0.2.0 ─────────────────────\n\nOLLAMA_HOST: str = os.getenv(\n \"OLLAMA_HOST\", resolve_setting(\"OLLAMA_HOST\", \"http://localhost:11434\")\n)\n\"\"\"Local Ollama server URL. Default matches Ollama's out-of-the-box\n``ollama serve`` binding. Set in .env or Settings dialog to point at a\ndifferent host (e.g. another machine on LAN). v0.2.0 only supports\nunauthenticated local Ollama — no API-key field needed.\"\"\"\n\nOLLAMA_MODEL_VISION: str = os.getenv(\n \"OLLAMA_MODEL_VISION\",\n resolve_setting(\"OLLAMA_MODEL_VISION\", \"llava:7b\"),\n)\n\"\"\"Ollama vision-capable model used when screenshots are present.\nDefault ``llava:7b`` works on every Ollama version with vision support\n(~4.5 GB). ``llama3.2-vision`` is more accurate but needs Ollama\n>=0.4.x (uses ``mllama`` arch). User can switch via Settings dialog;\n``ollama_health.check_model_compatibility`` warns on mismatch.\"\"\"\n\nOLLAMA_MODEL_TEXT: str = os.getenv(\n \"OLLAMA_MODEL_TEXT\",\n resolve_setting(\"OLLAMA_MODEL_TEXT\", \"llama3.2\"),\n)\n\"\"\"Ollama text-only model used when no screenshots are sent (rare in\nClicky's PTT flow but kept for parity with Bitshank's vision/text split).\nDefaults to plain ``llama3.2`` (3B, ~2 GB).\"\"\"\n\n\n# ── OpenAI (native API — gpt-5.4 vision + GPT-Realtime) — added v0.3.0 ───────\n\nOPENAI_MODEL_VISION: str = os.getenv(\n \"OPENAI_MODEL_VISION\",\n resolve_setting(\"OPENAI_MODEL_VISION\", \"gpt-5.4\"),\n)\n# Default bumped gpt-4o → gpt-5.4 (2026-06-25). Live-verified: on the EduPack\n# Chart/Select target, gpt-5.4 returned (258,53) vs Claude's (253,52) — 5px off,\n# pixel-perfect; gpt-4o returned (327,57) — 74px off. gpt-5.4 scores 85.4% on\n# ScreenSpot-Pro GUI grounding vs gpt-4o's weak coords. So the OpenAI vision\n# path no longer needs the grid-locator crutch (it auto-skips when a [POINT]\n# tag is returned). Env OPENAI_MODEL_VISION still overrides (e.g. =gpt-4o).\n\"\"\"OpenAI vision model for the normal pipeline (LLM_PROVIDER='openai').\n``gpt-4o`` by default — strong general vision, emits [POINT:x,y:label] via\nthe same Clicky system prompt as Claude. GPT-4o is weaker than Claude at\nraw pixel coordinates, so pointing is refined through the two-stage\ngrid-locator (locator.py), same pattern used for Ollama's weak vision\nmodels. Routed via the ``openai/`` MODEL_ID prefix in create_ai_client.\"\"\"\n\nOPENAI_REALTIME_MODEL: str = os.getenv(\n \"OPENAI_REALTIME_MODEL\",\n resolve_setting(\"OPENAI_REALTIME_MODEL\", \"gpt-realtime-2\"),\n)\n\"\"\"OpenAI GPT-Realtime model for the speech-to-speech path\n(LLM_PROVIDER='openai-realtime'). ``gpt-realtime-2`` is GPT-5-class,\ncontinuous-stream voice — near-zero latency, sees the screenshot, reasons,\nand emits a pointing target via the point_at function call. This path\nbypasses the STT→AIClient→TTS chain entirely (realtime.py owns the\nWebSocket session + audio I/O). Coordinates are refined via the\ngrid-locator, same as the GPT-4o path.\"\"\"\n\n\n# -- Local STT (faster-whisper, opt-in offline, no API key) ------------------\n\nFASTER_WHISPER_MODEL: str = os.getenv(\n \"FASTER_WHISPER_MODEL\", resolve_setting(\"FASTER_WHISPER_MODEL\", \"base.en\")\n)\n\"\"\"faster-whisper model size. 'base.en' is the low-latency English default\n(~150MB, downloads to the HF cache on first use). 'small.en' is more accurate\nbut slower. Local offline STT needs no API key.\"\"\"\n\nFASTER_WHISPER_DEVICE: str = os.getenv(\n \"FASTER_WHISPER_DEVICE\", resolve_setting(\"FASTER_WHISPER_DEVICE\", \"cpu\")\n)\n\"\"\"'cpu' (portable default) or 'cuda' if the user has an NVIDIA GPU.\"\"\"\n\nFASTER_WHISPER_COMPUTE: str = os.getenv(\n \"FASTER_WHISPER_COMPUTE\", resolve_setting(\"FASTER_WHISPER_COMPUTE\", \"int8\")\n)\n\"\"\"CTranslate2 compute type. 'int8' is fast + low-memory on CPU.\"\"\"\n\n\n# -- Local TTS (Kokoro-82M via ONNX, opt-in offline, no API key) -------------\n\nKOKORO_VOICE: str = os.getenv(\"KOKORO_VOICE\", resolve_setting(\"KOKORO_VOICE\", \"af_heart\"))\n\"\"\"Kokoro voice id. 'af_heart' is a warm conversational female voice.\"\"\"\n\nKOKORO_OUTPUT_SAMPLE_RATE: int = 24_000\n\"\"\"Kokoro-82M output sample rate (24kHz float32).\"\"\"\n\n_DEFAULT_KOKORO_DIR = Path.home() / \".clicky-windows\" / \"kokoro\"\nKOKORO_CACHE_DIR: Path = Path(os.getenv(\"KOKORO_CACHE_DIR\", str(_DEFAULT_KOKORO_DIR)))\n\"\"\"Where the Kokoro onnx + voices files download on first use (~336MB total).\"\"\"\n\n\n# -- Google Gemini (re-enabled cloud vision via OpenRouter) ------------------\n\nGEMINI_API_KEY: str | None = resolve_api_key(\"GEMINI_API_KEY\")\n\"\"\"OpenRouter key (sk-or-) for Gemini. Get one at https://openrouter.ai/keys.\nGemini routes through OpenRouter's OpenAI-compat endpoint (see GeminiClient).\"\"\"\n\nGEMINI_MODEL_VISION: str = os.getenv(\n \"GEMINI_MODEL_VISION\",\n resolve_setting(\"GEMINI_MODEL_VISION\", \"google/gemini-3.1-pro-preview\"),\n)\n\"\"\"Gemini model. Default is 'google/gemini-3.1-pro-preview' — the most\npixel-accurate Gemini for pointing (84.4% ScreenSpot-Pro, the Pro tier).\nv0.4.1 bumped the default off 'google/gemini-3.5-flash' because Flash's\ncoordinates were noticeably off in real use; there is no 3.5-pro, so the\n3.1 Pro preview is the strongest grounding option on OpenRouter. The full\n'-preview' suffix is the valid OpenRouter slug (bare 'google/gemini-3.1-pro'\n404s). Cheaper/faster alternative via env: GEMINI_MODEL_VISION=google/gemini-3.5-flash.\"\"\"\n\n\n# ── Memory ───────────────────────────────────────────────────────────────────\n\n_DEFAULT_MEMORY_DIR = Path.home() / \".clicky-windows\"\n\nMEMORY_DIR: Path = Path(os.getenv(\"MEMORY_DIR\", str(_DEFAULT_MEMORY_DIR / \"memory\")))\n\"\"\"Where per-app markdown files live. One .md per Windows app executable.\"\"\"\n\nINDEX_DB_PATH: Path = Path(os.getenv(\"INDEX_DB_PATH\", str(_DEFAULT_MEMORY_DIR / \"index.db\")))\n\"\"\"SQLite index at ~/.clicky-windows/index.db. Fast lookup for apps + interaction counts.\"\"\"\n\nINSIGHTS_PATH: Path = Path(os.getenv(\"INSIGHTS_PATH\", str(_DEFAULT_MEMORY_DIR / \"insights.md\")))\n\"\"\"Output of tools/lint_memory.py — Karpathy-style weekly health check.\"\"\"\n\nMEMORY_RECALL_MAX_CHARS: int = 1500\n\"\"\"Max characters of recalled memory to inject into the user message per request.\n~1500 chars = last 5-6 interactions. Clicky macOS sends zero persistent memory —\nour memory is the differentiator but too much context slows Claude down.\"\"\"\n\n\n# ── Knowledge base (user-uploadable per-app curated docs) ────────────────────\n\nKB_DIR: Path = Path(\n os.getenv(\"KB_DIR\", str(Path.home() / \"Documents\" / \"Clicky Wiki\"))\n)\n\"\"\"User drops a single .md file here per app, named to match the .exe\nbasename (e.g. ``edupack.exe.md`` for EduPack, ``fusion360.exe.md`` for\nFusion 360). Clicky reads it on every PTT and injects as authoritative\nreference in Claude's system prompt.\n\nDefault location is visible in File Explorer (NOT a hidden ``.``-prefixed\nfolder) so users can find + edit + delete the files without terminal\ngymnastics. Mirrors memory.py's transparency contract: human-readable,\nhand-editable, no vector DB.\n\nDecided 2026-05-04 after retracting JaySmith502's folder+TOML+section\npattern as cargo-cult for our scale. See DECISIONS.md.\"\"\"\n\nKB_RECALL_MAX_CHARS: int = 60_000\n\"\"\"Max characters of curated KB content to inject per request. ~15K\ntokens, ~⅓ of Claude's context budget. Over-budget files tail-truncate\n(same behavior as memory.recall). Anthropic supports up to 4\n``cache_control`` breakpoints per request; injecting KB adds a 2nd\nsystem block alongside the persona block, leaving 2 slots for the\nuser-message memory prefix + the implicit automatic-cache slot.\"\"\"\n\n\n# ── Overlay ──────────────────────────────────────────────────────────────────\n\nPOINTER_ANIMATION_MS: int = 400\n\"\"\"QPropertyAnimation duration for pointer movement. 400ms feels responsive,\nnot jittery. Phase 2 may switch to bezier easing.\"\"\"\n\n\n# ── Latency targets ──────────────────────────────────────────────────────────\n\nE2E_LATENCY_BUDGET_S: float = 1.5\n\"\"\"Target perceived latency from hotkey release to first audible word.\nExpected breakdown: ~150ms STT (AssemblyAI ForceEndpoint) + ~500-800ms\nClaude Sonnet 4.6 TTFT + ~200ms Cartesia Sonic-3 TTFB - ~300ms sentence-\nstreaming overlap = ~800-1200ms. See DECISIONS.md 'Priority inversion:\nlatency > local-first' for the full budget derivation.\"\"\"\n"}}, "ai.py": {"04f221f91d4750363d7032bcfd355b7dbc5c0e06": {"t": 1785674357.2098157, "n": 52106, "h": "939f3076a593387c", "c": "\"\"\"Clicky Windows vision+LLM layer.\n\nAIClient abstract base + AnthropicClient concrete implementation using\nplain vision streaming with a [POINT:x,y:label] coordinate tag regex parser.\nMatches Clicky's actual shipping path: ClaudeAPI.analyzeImageStreaming +\nCompanionManager.parsePointingCoordinates (verified line-by-line via gh api\non 2026-04-12). See DECISIONS.md 2026-04-12 (evening 3) for the research pass.\n\nResponsibility boundary:\n- THIS MODULE lives in Space C (Claude's declared resolution). It returns\n coordinates unclamped in that space.\n- capture.unscale_claude_coords() maps Space C -> Space A (physical pixels).\n- app.py owns threading and calls this module from worker threads.\n\nTop-to-bottom order (so `python -m ai` works):\n 1. Module docstring\n 2. Imports\n 3. Constants (_CLICKY_SYSTEM_PROMPT, _POINT_TAG_RE, _CLICKY_MAX_TOKENS)\n 4. PointParseResult dataclass\n 5. Pure functions (parse_point_tag, image_to_base64_jpeg, _get,\n parse_response_text)\n 6. AIClient abstract base class\n 7. AnthropicClient concrete class\n 8. __main__ block for manual live-API verification\n\"\"\"\nfrom __future__ import annotations\n\nimport base64\nimport re\nfrom abc import ABC, abstractmethod\nfrom dataclasses import dataclass\nfrom io import BytesIO\nfrom typing import Iterator\n\nimport httpx\nfrom anthropic import Anthropic\nfrom openai import OpenAI\nfrom PIL import Image\n\n\n# --- Constants ----------------------------------------------------------------\n\n_CLICKY_SYSTEM_PROMPT = \"\"\"\\\nyou're clicky, a friendly always-on companion that lives in the user's menu bar. the user just spoke to you via push-to-talk and you can see their screen(s). your reply will be spoken aloud via text-to-speech, so write the way you'd actually talk. this is an ongoing conversation — you remember everything they've said before.\n\nrules:\n- default to one or two sentences. be direct and dense. BUT if the user asks you to explain more, go deeper, or elaborate, then go all out — give a thorough, detailed explanation with no length limit.\n- all lowercase, casual, warm. no emojis.\n- write for the ear, not the eye. short sentences. no lists, bullet points, markdown, or formatting — just natural speech.\n- don't use abbreviations or symbols that sound weird read aloud. write \"for example\" not \"e.g.\", spell out small numbers.\n- if the user's question relates to what's on their screen, reference specific things you see.\n- if the screenshot doesn't seem relevant to their question, just answer the question directly.\n- you can help with anything — coding, writing, general knowledge, brainstorming.\n- never say \"simply\" or \"just\".\n- don't read out code verbatim. describe what the code does or what needs to change conversationally.\n- focus on giving a thorough, useful explanation. don't end with simple yes/no questions like \"want me to explain more?\" or \"should i show you?\" — those are dead ends that force the user to just say yes.\n- instead, when it fits naturally, end by planting a seed — mention something bigger or more ambitious they could try, a related concept that goes deeper, or a next-level technique that builds on what you just explained. make it something worth coming back for, not a question they'd just nod to. it's okay to not end with anything extra if the answer is complete on its own.\n- if you receive multiple screen images, the one labeled \"primary focus\" is where the cursor is — prioritize that one but reference others if relevant.\n\nelement pointing:\nyou have a small blue cursor that can fly to and point at things on screen. use it whenever pointing would genuinely help the user — if they're asking how to do something, looking for a menu, trying to find a button, or need help navigating an app, point at the relevant element. err on the side of pointing rather than not pointing, because it makes your help way more useful and concrete.\n\ndon't point at things when it would be pointless — like if the user asks a general knowledge question, or the conversation has nothing to do with what's on screen, or you'd just be pointing at something obvious they're already looking at. but if there's a specific UI element, menu, button, or area on screen that's relevant to what you're helping with, point at it.\n\nwhen you point, append a coordinate tag at the very end of your response, AFTER your spoken text. the screenshot images are labeled with their pixel dimensions. use those dimensions as the coordinate space. the origin (0,0) is the top-left corner of the image. x increases rightward, y increases downward.\n\nformat: [POINT:x,y:label] where x,y are integer pixel coordinates in the screenshot's coordinate space, and label is a short 1-3 word description of the element (like \"search bar\" or \"save button\"). if the element is on the cursor's screen you can omit the screen number. if the element is on a DIFFERENT screen, append :screenN where N is the screen number from the image label (e.g. :screen2). this is important — without the screen number, the cursor will point at the wrong place.\n\nif pointing wouldn't help, append [POINT:none].\n\nexamples:\n- user asks how to color grade in final cut: \"you'll want to open the color inspector — it's right up in the top right area of the toolbar. click that and you'll get all the color wheels and curves. [POINT:1100,42:color inspector]\"\n- user asks what html is: \"html stands for hypertext markup language, it's basically the skeleton of every web page. curious how it connects to the css you're looking at? [POINT:none]\"\n- user asks how to commit in xcode: \"see that source control menu up top? click that and hit commit, or you can use command option c as a shortcut. [POINT:285,11:source control]\"\n- element is on screen 2 (not where cursor is): \"that's over on your other monitor — see the terminal window? [POINT:400,300:terminal:screen2]\"\\\n\"\"\"\n\n_CLICKY_ANNOTATION_SYSTEM_PROMPT = \"\"\"\\\nyou're clicky, a patient tutor that teaches by drawing on the user's screen. the user just spoke to you via push-to-talk and you can see their screen(s). your reply is spoken aloud via text-to-speech, so write the way you'd actually talk — all lowercase, casual, warm, one or two short sentences (longer only if they ask you to go deeper). write for the ear: no lists, no markdown, no symbols that sound weird read aloud.\n\ninstead of only pointing with the cursor, you can DRAW on the screen to teach — circle the exact thing you mean, draw an arrow from a mistake to where it should go, underline a term, or write a short correction. use drawing whenever it makes your help more concrete: finding a button, explaining a step, showing a mistake on the user's work.\n\nhow to draw: explain in spoken words first, then append shape tags at the very END of your reply, after your spoken text. the screenshot images are labeled with their pixel dimensions — use those dimensions as the coordinate space. the origin (0,0) is the top-left corner of the image, x increases rightward, y increases downward. point at the SPECIFIC element, never a vague region. never read coordinates aloud. only annotate the screen labeled \"primary focus\" (the one with the cursor) — do NOT emit shape tags for elements on any other screen.\n\nshape tags (use as many as help, only what helps):\n- [CIRCLE:x,y,r:label] — circle the exact element. (x,y) is the center, r is the radius in pixels, label is a 1-3 word description.\n- [ARROW:x1,y1->x2,y2] — an arrow from (x1,y1) to (x2,y2), e.g. from the mistake to where it should go.\n- [UNDERLINE:x,y,w] — underline a term or formula. (x,y) is the left end, w is the width in pixels.\n- [LABEL:x,y:text] — write a short correction or hint on the screen at (x,y).\n\nif nothing on screen is relevant (a purely conceptual question), just speak with no tags.\n\nexamples:\n- user asks why their answer is wrong: \"you applied the power rule but missed the chain rule here — you need to multiply by the derivative of the inside. [CIRCLE:340,210,28:missing step][ARROW:340,210->410,260]\"\n- user asks where to save: \"hit the save button up in the top left of the toolbar. [CIRCLE:48,30,18:save]\"\n- user asks what a binary tree is (nothing on screen): \"it's a structure where each node has at most two children, a left and a right. great for fast lookups.\"\\\n\"\"\"\n\"\"\"Annotation-mode system prompt (hackathon draw-on-screen). Same\ndimension-labeled coordinate-space contract as the cursor [POINT] prompt\n(Farza CompanionManager.swift:566), extended from a single point to four\nshape tags parsed by annotations.parse_annotations. Used when\nconfig.ANNOTATION_MODE == 'on'.\"\"\"\n\n_POINT_TAG_RE = re.compile(\n r\"\\[POINT:(?:none|(\\d+)\\s*,\\s*(\\d+)(?::(?!screen\\d)([^\\]:\\s][^\\]:]*?))?(?::screen(\\d+))?)\\]\\s*$\"\n)\n\"\"\"Regex for Clicky's [POINT:x,y:label(:screenN)?] coordinate tag.\n\nPython port of CompanionManager.parsePointingCoordinates\n(leanring-buddy/CompanionManager.swift:784-828).\n\"\"\"\n\n_CLICKY_MAX_TOKENS = 1024\n\"\"\"Token budget matching Clicky's analyzeImageStreaming call.\"\"\"\n\n_MEMORY_PREFIX_MARKER = \"[context from past sessions\"\n\"\"\"Sentinel that app.py prepends to the user transcript when memory is\ninjected. Used by AnthropicClient.ask_stream to split the transcript into a\ncached memory-prefix block + an uncached current-turn block. Must match\napp.py ClickyApp._pipeline_worker's f-string exactly.\"\"\"\n\n_KB_SYSTEM_PREFIX_TEMPLATE = (\n \"app knowledge base:\\n\"\n \"you are helping the user with {app_name}. here is reference \"\n \"documentation that you should treat as authoritative:\\n\\n\"\n)\n\"\"\"Marker prefix prepended to user-uploaded KB content before injection\ninto the system prompt as a SECOND cache_control block. Caller (app.py\n_pipeline_worker → ask_stream's kb_content kwarg) supplies the raw\nmarkdown body; ask_stream formats this prefix in front and adds the\nephemeral cache breakpoint. Per-app cache hit on subsequent turns within\nthe same app session; cache miss on app switch (acceptable since each\nKB read is the dominant cost anyway). Empty kb_content means no second\nblock — Claude proceeds with vision + memory only (the 'Claude already\nknows that software' path).\"\"\"\n\n\n# --- PointParseResult ---------------------------------------------------------\n\n@dataclass\nclass PointParseResult:\n \"\"\"Result of parsing the [POINT:...] tag from Claude's response text.\"\"\"\n spoken_text: str\n coordinate: tuple[int, int] | None\n element_label: str | None\n screen_number: int | None\n\n\n# --- Pure functions -----------------------------------------------------------\n\ndef parse_point_tag(text: str) -> PointParseResult:\n \"\"\"Extract coordinate from a trailing [POINT:x,y:label] tag and strip it.\n\n Returns PointParseResult with coordinate=None on [POINT:none] or no match.\n The spoken_text field has the tag removed so TTS never reads it aloud.\n \"\"\"\n match = _POINT_TAG_RE.search(text)\n if not match:\n return PointParseResult(\n spoken_text=text.strip(),\n coordinate=None,\n element_label=None,\n screen_number=None,\n )\n\n spoken = _POINT_TAG_RE.sub(\"\", text).strip()\n\n if match.group(1) is None:\n return PointParseResult(\n spoken_text=spoken,\n coordinate=None,\n element_label=None,\n screen_number=None,\n )\n\n x, y = int(match.group(1)), int(match.group(2))\n label = match.group(3)\n screen = int(match.group(4)) if match.group(4) else None\n\n return PointParseResult(\n spoken_text=spoken,\n coordinate=(x, y),\n element_label=label,\n screen_number=screen,\n )\n\n\ndef image_to_base64_jpeg(img: Image.Image, quality: int = 85) -> str:\n \"\"\"Encode a PIL image to a base64-ASCII JPEG string for the Claude API.\"\"\"\n buf = BytesIO()\n img.save(buf, \"JPEG\", quality=quality)\n return base64.b64encode(buf.getvalue()).decode(\"ascii\")\n\n\ndef _get(obj, key, default=None):\n \"\"\"Dual-access helper: works on both dict-shaped test mocks and\n anthropic SDK objects (via attribute access).\"\"\"\n if isinstance(obj, dict):\n return obj.get(key, default)\n return getattr(obj, key, default)\n\n\ndef parse_response_text(response) -> str:\n \"\"\"Concatenate all text-type content blocks into a single string.\n\n Dual-access compatible (dict mocks or SDK objects). Used by the batch\n ask() wrapper to extract the full text from a non-streaming response.\n \"\"\"\n content = _get(response, \"content\", []) or []\n texts: list[str] = []\n for block in content:\n if _get(block, \"type\") != \"text\":\n continue\n text = _get(block, \"text\", \"\") or \"\"\n if text:\n texts.append(text)\n return \" \".join(texts).strip()\n\n\n# --- AIClient abstract base ---------------------------------------------------\n\nclass AIClient(ABC):\n \"\"\"Abstract base for vision+LLM providers.\n\n Phase 1: AnthropicClient (vision-tag streaming).\n Phase 2: OpenRouterClient, GeminiClient, etc. as subclass drops.\n \"\"\"\n\n @abstractmethod\n def ask(\n self,\n image: Image.Image,\n transcript: str,\n history: list[dict],\n declared_w: int,\n declared_h: int,\n ) -> dict:\n \"\"\"Return {\"text\": str, \"points\": [{\"x\":int,\"y\":int,\"label\":str}]}.\n\n Coordinates are in Claude's declared-resolution space (Space C),\n unclamped. Caller uses capture.unscale_claude_coords() to map to\n physical pixels (Space A).\n \"\"\"\n ...\n\n\n# --- Concrete Anthropic implementation ----------------------------------------\n\nclass AnthropicClient(AIClient):\n \"\"\"Phase 1 implementation using plain vision streaming + [POINT:x,y:label].\n\n Matches Clicky's actual shipping path: ClaudeAPI.analyzeImageStreaming +\n CompanionManager.parsePointingCoordinates. NOT Computer Use API beta\n (that was dead code in Clicky — ElementLocationDetector.swift, 0 refs).\n See DECISIONS.md 2026-04-12 (evening 3).\n \"\"\"\n\n def __init__(\n self,\n api_key: str,\n model_id: str,\n base_url: str | None = None,\n ) -> None:\n kwargs: dict = {\"api_key\": api_key, \"timeout\": 60.0}\n if base_url is not None:\n kwargs[\"base_url\"] = base_url\n self.client = Anthropic(**kwargs)\n self.model_id = model_id\n\n def ask_stream(\n self,\n images: list[tuple[Image.Image, str]],\n transcript: str,\n history: list[dict],\n system_prompt: str = _CLICKY_SYSTEM_PROMPT,\n max_tokens: int = _CLICKY_MAX_TOKENS,\n kb_content: str = \"\",\n kb_app_name: str = \"\",\n ):\n \"\"\"Open a streaming Claude call, return a context manager.\n\n Args:\n images: list of (PIL Image, label string) tuples — one per screen.\n Sorted cursor-screen-first by capture_all_screens(). Each\n becomes an image content block + a text label block in the\n user message. Matches Clicky's analyzeImageStreaming(images:\n [(Data, String)], ...) shape.\n transcript: user's voice question (raw STT output).\n history: prior turns in Anthropic SDK message format.\n system_prompt: persona + pointing instructions.\n max_tokens: token budget (1024 default, matches Clicky).\n kb_content: optional curated KB markdown body (from\n kb.recall). If non-empty, injected as a SECOND\n cache_control system block alongside the persona block.\n Empty (default) → only the persona block is sent.\n kb_app_name: sanitized .exe basename used to format the KB\n injection marker (e.g. \"edupack.exe\" → display \"edupack\").\n Ignored when kb_content is empty.\n\n Usage:\n with client.ask_stream(images, transcript, history) as stream:\n for delta in stream.text_deltas():\n # progressive text for sentence-level TTS chunking\n pass\n result = stream.final_result()\n # result.spoken_text, result.coordinate, etc.\n \"\"\"\n content_blocks: list[dict] = []\n for img, label in images:\n base64_jpeg = image_to_base64_jpeg(img)\n content_blocks.append({\n \"type\": \"image\",\n \"source\": {\n \"type\": \"base64\",\n \"media_type\": \"image/jpeg\",\n \"data\": base64_jpeg,\n },\n })\n content_blocks.append({\"type\": \"text\", \"text\": label})\n\n # Split the user transcript into a cached memory-prefix block + an\n # uncached current-transcript block when the memory marker is present.\n # app.py (Step 7 pipeline worker) prepends memory context as:\n # \"[context from past sessions ...]\\n<memory>\\n\\n<actual transcript>\"\n # Caching the prefix saves ~50-100ms TTFT after the first hit (5-min\n # TTL). NEVER cache the current transcript — per-turn content is what\n # makes the full-context-caching latency paradox bite (arxiv 2601.06007\n # \"Don't Break the Cache\" — only stable prefixes help).\n if _MEMORY_PREFIX_MARKER in transcript:\n parts = transcript.split(\"\\n\\n\", 1)\n if len(parts) == 2:\n memory_text, actual_transcript = parts\n content_blocks.append({\n \"type\": \"text\",\n \"text\": memory_text + \"\\n\\n\",\n \"cache_control\": {\"type\": \"ephemeral\"},\n })\n content_blocks.append({\"type\": \"text\", \"text\": actual_transcript})\n else:\n content_blocks.append({\"type\": \"text\", \"text\": transcript})\n else:\n content_blocks.append({\"type\": \"text\", \"text\": transcript})\n\n new_user_turn = {\"role\": \"user\", \"content\": content_blocks}\n\n # Cache the system prompt (largest stable text block, ~1500 chars).\n # OpenRouter passes Anthropic-native cache_control through for\n # anthropic/* routes per openrouter.ai/docs/guides/best-practices/prompt-caching.\n system_blocks = [\n {\n \"type\": \"text\",\n \"text\": system_prompt,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n ]\n # Optional second cache_control block for user-uploaded curated KB\n # (kb.recall result). Per-app cache: hit within same app session,\n # miss on app switch. Anthropic's 4-block limit accommodates this\n # plus the user-message memory prefix block.\n if kb_content:\n display_name = kb_app_name.removesuffix(\".exe\") or \"this software\"\n kb_text = (\n _KB_SYSTEM_PREFIX_TEMPLATE.format(app_name=display_name)\n + kb_content\n )\n system_blocks.append({\n \"type\": \"text\",\n \"text\": kb_text,\n \"cache_control\": {\"type\": \"ephemeral\"},\n })\n\n sdk_stream_mgr = self.client.messages.stream(\n model=self.model_id,\n max_tokens=max_tokens,\n system=system_blocks,\n messages=[*history, new_user_turn],\n )\n\n return _StreamingResponse(sdk_stream_mgr)\n\n def ask(\n self,\n image: Image.Image,\n transcript: str,\n history: list[dict],\n declared_w: int,\n declared_h: int,\n ) -> dict:\n \"\"\"Batch wrapper: consumes the full stream, returns parsed dict.\n\n Wraps a single image into the list format ask_stream() expects.\n Backwards-compatible with the __main__ gate and test shapes.\n \"\"\"\n label = f\"primary focus (image dimensions: {declared_w}x{declared_h} pixels)\"\n with self.ask_stream(\n [(image, label)], transcript, history\n ) as stream:\n for _ in stream.text_deltas():\n pass\n result = stream.final_result()\n\n points = []\n if result.coordinate:\n x, y = result.coordinate\n points.append({\"x\": x, \"y\": y, \"label\": result.element_label or \"\"})\n\n return {\"text\": result.spoken_text, \"points\": points}\n\n\nclass _StreamingResponse:\n \"\"\"Wraps the SDK's MessageStreamManager for Clicky's streaming pattern.\"\"\"\n\n def __init__(self, sdk_stream_mgr):\n self._sdk_mgr = sdk_stream_mgr\n self._sdk_stream = None\n self._accumulated = \"\"\n self._deltas_exhausted = False\n\n def __enter__(self):\n self._sdk_stream = self._sdk_mgr.__enter__()\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n return self._sdk_mgr.__exit__(exc_type, exc_val, exc_tb)\n\n def text_deltas(self) -> Iterator[str]:\n \"\"\"Yield progressive text deltas for sentence-level TTS chunking.\"\"\"\n for delta in self._sdk_stream.text_stream:\n self._accumulated += delta\n yield delta\n self._deltas_exhausted = True\n\n def final_result(self) -> PointParseResult:\n \"\"\"Parse the accumulated text for a [POINT:x,y:label] tag.\n\n If text_deltas() was fully exhausted, uses the accumulated text.\n Otherwise falls back to get_final_text() which blocks until the\n stream completes.\n \"\"\"\n if not self._deltas_exhausted:\n self._accumulated = self._sdk_stream.get_final_text()\n return parse_point_tag(self._accumulated)\n\n\n# --- Gemini (via OpenRouter OpenAI-compat endpoint) --------------------------\n\nclass GeminiClient(AIClient):\n \"\"\"Gemini 3 Flash Preview (or any OpenRouter google/* model) via the\n OpenAI Python SDK pointed at OpenRouter's OpenAI-compat endpoint.\n\n Response parsing (parse_point_tag) is identical — both Claude and Gemini\n emit [POINT:x,y:label] per the verbatim Clicky system prompt. The only\n differences vs AnthropicClient are request shape (OpenAI chat.completions\n format instead of Anthropic messages format) and image block format\n (image_url with data URL instead of base64 source block).\n\n History assumption: we convert Anthropic content-block format to plain\n strings by concatenating text blocks. Non-text blocks in history are\n dropped. Phase 1 history only contains text blocks so this is safe; if\n Phase 2 adds image-bearing history, revisit this.\n\n See DECISIONS.md 2026-04-19 'Gemini 3 Flash via OpenRouter' for the\n dual-SDK routing rationale.\n \"\"\"\n\n # OpenRouter accepts the classic `max_tokens` for every model + normalizes\n # it. A provider's NATIVE endpoint may need a different param (OpenAI's\n # gpt-5-series requires `max_completion_tokens`), so subclasses override\n # this; it is used only on the native (non-OpenRouter) path.\n _native_token_param: str = \"max_tokens\"\n\n def __init__(\n self,\n api_key: str,\n model_id: str,\n base_url: str = \"https://openrouter.ai/api/v1\",\n ) -> None:\n # OpenRouter wants the NAMESPACED slug (google/..., openai/...); a\n # provider's native OpenAI-compatible endpoint wants the BARE name, so\n # strip the prefix. Detect which by the base_url.\n via_openrouter = bool(base_url) and \"openrouter.ai\" in base_url\n if via_openrouter:\n self.model_id = model_id\n else:\n self.model_id = model_id.split(\"/\", 1)[1] if \"/\" in model_id else model_id\n self.client = OpenAI(api_key=api_key, base_url=base_url, timeout=60.0)\n self._max_tokens_param = (\n \"max_tokens\" if via_openrouter else self._native_token_param\n )\n\n def ask_stream(\n self,\n images: list[tuple[Image.Image, str]],\n transcript: str,\n history: list[dict],\n system_prompt: str = _CLICKY_SYSTEM_PROMPT,\n max_tokens: int = _CLICKY_MAX_TOKENS,\n kb_content: str = \"\",\n kb_app_name: str = \"\",\n ):\n \"\"\"Open a streaming Gemini call. Returns a context manager with the\n same interface as AnthropicClient.ask_stream().\n\n Builds OpenAI-shaped messages:\n - System prompt goes as messages[0] role=system. If\n ``kb_content`` is non-empty, the KB block is concatenated\n onto the system prompt (Gemini via OpenAI-compat doesn't\n support multiple system blocks or cache_control breakpoints,\n so caching is best-effort via OpenRouter's prompt-caching\n auto-detection).\n - History is converted from Anthropic content-block format to\n OpenAI plain-string content (text blocks are concatenated)\n - Current user turn gets image_url + text blocks\n\n Usage is identical to AnthropicClient:\n with client.ask_stream(images, transcript, history) as stream:\n for delta in stream.text_deltas():\n ...\n result = stream.final_result()\n \"\"\"\n user_content: list[dict] = []\n for img, label in images:\n base64_jpeg = image_to_base64_jpeg(img)\n user_content.append({\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": f\"data:image/jpeg;base64,{base64_jpeg}\",\n },\n })\n user_content.append({\"type\": \"text\", \"text\": label})\n user_content.append({\"type\": \"text\", \"text\": transcript})\n\n # Concat KB into system prompt for Gemini (no native multi-block\n # support via OpenAI-compat endpoint).\n full_system = system_prompt\n if kb_content:\n display_name = kb_app_name.removesuffix(\".exe\") or \"this software\"\n full_system = (\n system_prompt\n + \"\\n\\n\"\n + _KB_SYSTEM_PREFIX_TEMPLATE.format(app_name=display_name)\n + kb_content\n )\n\n openai_messages: list[dict] = [\n {\"role\": \"system\", \"content\": full_system}\n ]\n for turn in history:\n text_parts = []\n for block in turn.get(\"content\", []):\n if isinstance(block, dict) and block.get(\"type\") == \"text\":\n text_parts.append(block.get(\"text\", \"\"))\n # Skip turns with no text content (e.g., image-only turns from\n # Phase 2, or malformed turns) — OpenRouter rejects empty content.\n if not any(p.strip() for p in text_parts):\n continue\n openai_messages.append({\n \"role\": turn[\"role\"],\n \"content\": \" \".join(text_parts),\n })\n openai_messages.append({\"role\": \"user\", \"content\": user_content})\n\n try:\n sdk_iterator = self.client.chat.completions.create(\n model=self.model_id,\n messages=openai_messages,\n stream=True,\n **{self._max_tokens_param: max_tokens},\n )\n except Exception as exc:\n raise RuntimeError(\n f\"Gemini request failed (model={self.model_id!r}). \"\n \"Diagnostic checklist:\\n\"\n \" 1. Is the OpenRouter key (ANTHROPIC_API_KEY in .env) valid + funded?\\n\"\n \" 2. Is the model available on your account? Preview models like \"\n \"'google/gemini-3-flash-preview' require opt-in at \"\n \"https://openrouter.ai/settings/privacy. Fallback: set \"\n \"MODEL_ID=google/gemini-2.5-flash in .env.\\n\"\n \" 3. Is your internet connection up?\\n\"\n f\"Underlying error: {type(exc).__name__}: {exc}\"\n ) from exc\n return _GeminiStreamingResponse(sdk_iterator)\n\n def ask(\n self,\n image: Image.Image,\n transcript: str,\n history: list[dict],\n declared_w: int,\n declared_h: int,\n ) -> dict:\n \"\"\"Batch wrapper for parity with AnthropicClient.ask().\"\"\"\n label = f\"primary focus (image dimensions: {declared_w}x{declared_h} pixels)\"\n with self.ask_stream([(image, label)], transcript, history) as stream:\n for _ in stream.text_deltas():\n pass\n result = stream.final_result()\n points = []\n if result.coordinate:\n x, y = result.coordinate\n points.append({\"x\": x, \"y\": y, \"label\": result.element_label or \"\"})\n return {\"text\": result.spoken_text, \"points\": points}\n\n\nclass _GeminiStreamingResponse:\n \"\"\"Wraps OpenAI SDK streaming iterator to match AnthropicClient's\n _StreamingResponse public interface (context manager + text_deltas() +\n final_result()). Consumers of ask_stream() don't need to know which\n client is behind it.\n \"\"\"\n\n def __init__(self, sdk_iterator):\n self._sdk_iterator = sdk_iterator\n self._accumulated = \"\"\n self._deltas_exhausted = False\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n try:\n self._sdk_iterator.close()\n except Exception:\n pass\n return False # don't swallow exceptions\n\n def text_deltas(self) -> Iterator[str]:\n for chunk in self._sdk_iterator:\n if not chunk.choices:\n continue\n delta = chunk.choices[0].delta.content\n if delta:\n self._accumulated += delta\n yield delta\n self._deltas_exhausted = True\n\n def final_result(self) -> PointParseResult:\n if not self._deltas_exhausted:\n for _ in self.text_deltas():\n pass\n return parse_point_tag(self._accumulated)\n\n\n# --- OpenAIVisionClient (v0.3.0 — GPT-4o via OpenAI native API) --------------\n\nclass OpenAIVisionClient(GeminiClient):\n \"\"\"OpenAI vision model (default gpt-5.4) via the OpenAI Python SDK pointed\n at OpenAI's native endpoint (api.openai.com).\n\n Reuses GeminiClient's ``ask_stream`` / ``ask`` / ``_GeminiStreamingResponse``\n verbatim — both are OpenAI-chat-completions-shaped, the only difference is\n the endpoint + key. Emits ``[POINT:x,y:label]`` per the same Clicky system\n prompt.\n\n Accuracy note (2026-06-25): the default model is now **gpt-5.4**, which is\n pixel-accurate at grounding (live-verified 5px off Claude on the EduPack\n target; 85.4% ScreenSpot-Pro). It returns a precise ``[POINT]`` tag directly,\n so app.py's grid-locator auto-skips (it only fires when no tag is present).\n The grid-locator crutch was for the OLD gpt-4o default (74px off) and for\n Ollama's weak local vision models — both still supported via env override.\n\n Routed via the ``openai/`` MODEL_ID prefix in create_ai_client(). The\n GPT-Realtime speech-to-speech path is separate (realtime.py); its accurate\n pixel pass also goes through this client (gpt-5.4) — see app.py.\n \"\"\"\n\n # gpt-5-series rejects `max_tokens` on OpenAI's NATIVE endpoint and requires\n # `max_completion_tokens` (400 otherwise; v0.4.1 fix — gpt-5.4 was 400ing\n # every call → no audio). Via OpenRouter, max_tokens works (the base class\n # handles that). Used only on the native path.\n _native_token_param: str = \"max_completion_tokens\"\n\n def __init__(\n self,\n api_key: str,\n model_id: str,\n base_url: str | None = None,\n ) -> None:\n # Same dual routing as the GeminiClient base: OpenRouter keeps the\n # namespaced openai/ slug; a direct OpenAI key (base_url=None →\n # api.openai.com) strips to the bare model name. create_ai_client picks\n # base_url by key prefix via _provider_base_url.\n super().__init__(api_key=api_key, model_id=model_id, base_url=base_url)\n\n\n# --- OllamaClient (v0.2.0 local LLM support) ---------------------------------\n\nclass OllamaClient(AIClient):\n \"\"\"Local LLM via Ollama server (https://ollama.com).\n\n Speaks Ollama's /api/chat streaming protocol over HTTP. Accepts multi-image\n screenshots passed as base64-encoded JPEGs inline in the user message's\n ``images`` field (Ollama-specific extension to the OpenAI-style messages\n array).\n\n **Pixel-pointing caveat:** local vision models (llama3.2-vision, qwen2.5-vl,\n llava) generally cannot return precise pixel coordinates via free-text\n [POINT:x,y:label] tags. app.py wires a grid-locator fallback (locator.py)\n that runs AFTER the streamed response completes: if no [POINT:x,y] tag was\n emitted AND the query was directional, the locator pass derives coordinates\n via two-stage grid annotation. See DECISIONS.md 2026-06-05 (Sprint v0.2.0)\n for the rationale + Bitshank-2338/clicky-windows reference implementation.\n\n Public interface matches AnthropicClient + GeminiClient: ``ask_stream(...)``\n returns a context manager with .text_deltas() generator + .final_result()\n returning PointParseResult. app.py's _pipeline_worker doesn't need to know\n which client is behind it.\n \"\"\"\n\n def __init__(self, host: str, model_id: str) -> None:\n # Strip optional 'ollama/' prefix — Ollama API wants the bare model name.\n if model_id.lower().startswith(\"ollama/\"):\n self.model_id = model_id[len(\"ollama/\"):]\n else:\n self.model_id = model_id\n self.host = host.rstrip(\"/\")\n\n def ask_stream(\n self,\n images: list[tuple[Image.Image, str]],\n transcript: str,\n history: list[dict],\n system_prompt: str = _CLICKY_SYSTEM_PROMPT,\n max_tokens: int = _CLICKY_MAX_TOKENS,\n kb_content: str = \"\",\n kb_app_name: str = \"\",\n ):\n \"\"\"Open a streaming Ollama /api/chat call. Returns a context manager\n with the same interface as AnthropicClient.ask_stream().\n\n Builds Ollama-shaped messages:\n - System prompt as messages[0] role=system. KB block concat'd onto\n system prompt (Ollama doesn't support multi-block cache_control).\n - History flattened from Anthropic content-block format to plain\n strings (Ollama uses OpenAI-style string content).\n - Current user turn: {role: user, content: transcript, images: [b64...]}\n with base64-encoded JPEG screenshots as a list of strings (Ollama's\n extension to the standard message shape).\n\n Usage:\n with client.ask_stream(images, transcript, history) as stream:\n for delta in stream.text_deltas():\n ...\n result = stream.final_result()\n \"\"\"\n # Concat KB into system prompt (no multi-block support on Ollama).\n full_system = system_prompt\n if kb_content:\n display_name = kb_app_name.removesuffix(\".exe\") or \"this software\"\n full_system = (\n system_prompt\n + \"\\n\\n\"\n + _KB_SYSTEM_PREFIX_TEMPLATE.format(app_name=display_name)\n + kb_content\n )\n\n # Build OpenAI-style messages array.\n ollama_messages: list[dict] = [\n {\"role\": \"system\", \"content\": full_system}\n ]\n\n # Flatten history (Anthropic content-blocks → plain strings).\n for turn in history:\n text_parts: list[str] = []\n content = turn.get(\"content\", \"\")\n if isinstance(content, list):\n for block in content:\n if isinstance(block, dict) and block.get(\"type\") == \"text\":\n text_parts.append(block.get(\"text\", \"\"))\n elif isinstance(content, str):\n text_parts.append(content)\n # Skip empty turns (image-only or malformed) — Ollama rejects empty content.\n if not any(p.strip() for p in text_parts):\n continue\n ollama_messages.append({\n \"role\": turn[\"role\"],\n \"content\": \" \".join(text_parts),\n })\n\n # Current user turn — transcript as content, screenshots as images list\n user_turn: dict = {\"role\": \"user\", \"content\": transcript}\n if images:\n user_turn[\"images\"] = [\n image_to_base64_jpeg(img) for img, _label in images\n ]\n ollama_messages.append(user_turn)\n\n payload = {\n \"model\": self.model_id,\n \"messages\": ollama_messages,\n \"stream\": True,\n \"options\": {\n \"num_predict\": max_tokens,\n \"temperature\": 0.7,\n },\n }\n\n return _OllamaStreamingResponse(\n host=self.host,\n payload=payload,\n model_for_errors=self.model_id,\n )\n\n def ask(\n self,\n image: Image.Image,\n transcript: str,\n history: list[dict],\n declared_w: int,\n declared_h: int,\n ) -> dict:\n \"\"\"Batch wrapper for parity with AnthropicClient.ask() / GeminiClient.ask().\"\"\"\n label = f\"primary focus (image dimensions: {declared_w}x{declared_h} pixels)\"\n with self.ask_stream([(image, label)], transcript, history) as stream:\n for _ in stream.text_deltas():\n pass\n result = stream.final_result()\n points = []\n if result.coordinate:\n x, y = result.coordinate\n points.append({\"x\": x, \"y\": y, \"label\": result.element_label or \"\"})\n return {\"text\": result.spoken_text, \"points\": points}\n\n\nclass _OllamaStreamingResponse:\n \"\"\"Wraps httpx streaming response from Ollama /api/chat to match\n AnthropicClient + GeminiClient's _StreamingResponse public interface\n (context manager + text_deltas() + final_result()). Consumers of\n ask_stream() don't need to know which client is behind it.\n\n Each instance opens its own httpx.Client + stream context on __enter__\n (so the network call doesn't fire until the caller actually wants to\n stream). __exit__ closes both.\n \"\"\"\n\n def __init__(self, host: str, payload: dict, model_for_errors: str) -> None:\n self._host = host\n self._payload = payload\n self._model_for_errors = model_for_errors\n self._httpx_client = None\n self._stream_cm = None\n self._response = None\n self._accumulated = \"\"\n self._deltas_exhausted = False\n\n def __enter__(self):\n # Open the httpx client first. If any subsequent open-step raises,\n # we MUST close this client — the caller's `with` block never gets\n # entered, so __exit__ won't fire. Without the try/except below the\n # client would leak its connection pool every time Ollama is\n # unreachable (DNS failure, ECONNREFUSED, server crash, etc).\n # Caught by superpowers:code-reviewer 2026-06-05.\n self._httpx_client = httpx.Client(timeout=120.0)\n self._httpx_client.__enter__()\n try:\n self._stream_cm = self._httpx_client.stream(\n \"POST\",\n f\"{self._host}/api/chat\",\n json=self._payload,\n )\n self._response = self._stream_cm.__enter__()\n\n # Friendly error for the most common Ollama mistake: model not pulled\n if self._response.status_code == 404:\n # Close before raising to release the connection\n try:\n self._stream_cm.__exit__(None, None, None)\n finally:\n self._httpx_client.__exit__(None, None, None)\n self._httpx_client = None\n self._stream_cm = None\n raise RuntimeError(\n f\"Ollama doesn't have '{self._model_for_errors}' installed. \"\n f\"Run: ollama pull {self._model_for_errors}\"\n )\n self._response.raise_for_status()\n except BaseException:\n # Any failure between httpx_client open and successful\n # raise_for_status (network error, 5xx, etc.): close the\n # client we just opened, null the slot so __exit__ is a no-op,\n # re-raise so the caller sees the original exception.\n if self._stream_cm is not None:\n try:\n self._stream_cm.__exit__(None, None, None)\n except Exception:\n pass\n self._stream_cm = None\n try:\n self._httpx_client.__exit__(None, None, None)\n except Exception:\n pass\n self._httpx_client = None\n raise\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n try:\n if self._stream_cm is not None:\n self._stream_cm.__exit__(exc_type, exc_val, exc_tb)\n finally:\n if self._httpx_client is not None:\n self._httpx_client.__exit__(exc_type, exc_val, exc_tb)\n return False # don't swallow exceptions\n\n def text_deltas(self) -> Iterator[str]:\n \"\"\"Yield progressive text chunks from Ollama's JSON-per-line stream.\n\n Each line is a JSON object like {\"message\": {\"content\": \"...\"}, \"done\": false}.\n We accumulate the .message.content fields and yield non-empty chunks.\n\n On socket failure mid-stream (Ollama crash, network drop), httpx raises\n ReadError / RemoteProtocolError. We MUST mark deltas_exhausted=True in\n a finally block so that a subsequent final_result() call doesn't re-\n enter this iterator and re-raise the same exception, instead returning\n whatever was accumulated before the failure (graceful degradation —\n the user gets a partial response with parse_point_tag falling back to\n no coordinate). Caught by superpowers:code-reviewer 2026-06-05.\n \"\"\"\n import json as _json\n try:\n for line in self._response.iter_lines():\n if not line:\n continue\n # iter_lines() in httpx returns str by default in newer\n # versions, bytes in older. Handle both.\n if isinstance(line, bytes):\n line = line.decode(\"utf-8\", errors=\"replace\")\n try:\n data = _json.loads(line)\n except _json.JSONDecodeError:\n continue\n chunk = data.get(\"message\", {}).get(\"content\", \"\")\n if chunk:\n self._accumulated += chunk\n yield chunk\n if data.get(\"done\"):\n break\n finally:\n # ALWAYS set this — even on exception. Prevents final_result()\n # from re-entering this generator and re-raising the same error.\n self._deltas_exhausted = True\n\n def final_result(self) -> PointParseResult:\n \"\"\"Parse the accumulated text for a [POINT:x,y:label] tag.\n\n Safe to call even if text_deltas() raised mid-stream: we just parse\n whatever was accumulated before the failure (may be empty string,\n in which case parse_point_tag returns PointParseResult with no\n coordinate and empty spoken_text). The pipeline worker handles\n empty results gracefully.\n \"\"\"\n if not self._deltas_exhausted:\n # text_deltas() guarantees _deltas_exhausted=True even on failure,\n # so this re-entry can only happen if the caller never consumed\n # any deltas. Consume them now (best-effort — if iter_lines\n # raises, our own try/finally above flips the flag so we don't\n # loop forever).\n try:\n for _ in self.text_deltas():\n pass\n except Exception:\n # Already accumulated whatever streamed in; parse_point_tag\n # below will return a PointParseResult with no coordinate.\n pass\n return parse_point_tag(self._accumulated)\n\n\n# --- Factory: route model_id prefix to right AIClient subclass ---------------\n\n# Bare-name prefixes that route to OllamaClient (in addition to ollama/* prefix).\n# These match common local-model name conventions Ollama uses.\n_OLLAMA_BARE_PREFIXES = (\"llama\", \"qwen\", \"llava\", \"mistral\", \"phi\", \"gemma\")\n\n\ndef _provider_base_url(\n api_key: str,\n base_url: str | None,\n openrouter_url: str,\n native_url: str | None = None,\n) -> str | None:\n \"\"\"Shared endpoint decision for every LLM provider. An explicit base_url\n wins; else an OpenRouter (sk-or-) key routes to openrouter_url; else the\n provider's native endpoint (native_url, or None = the SDK's own default).\n\n Centralizing this is the point: no provider can 'forget' OpenRouter routing.\n OpenAI did, pre-v0.4.x — an sk-or- key was silently sent to api.openai.com\n and 401'd. Every branch in create_ai_client now calls this.\n \"\"\"\n if base_url is not None:\n return base_url\n if api_key and api_key.startswith(\"sk-or-\"):\n return openrouter_url\n return native_url\n\n\ndef create_ai_client(\n model_id: str,\n api_key: str,\n base_url: str | None = None,\n ollama_host: str | None = None,\n) -> AIClient:\n \"\"\"Route to AnthropicClient, GeminiClient, or OllamaClient based on model_id prefix.\n\n This is THE BYOK abstraction. Users change MODEL_ID in .env, app.py calls\n this factory, and the right SDK routes the request. No app.py logic\n depends on which model family is active.\n\n Args:\n model_id: OpenRouter-style model ID. Prefix determines client:\n 'anthropic/...' or 'claude...' → AnthropicClient (anthropic SDK)\n 'google/...' or 'gemini...' → GeminiClient (openai SDK)\n 'ollama/...' or bare 'llama*' / → OllamaClient (httpx → local server)\n 'qwen*' / 'llava*' / 'mistral*' /\n 'phi*' / 'gemma*'\n Other prefixes raise ValueError with an actionable message.\n api_key: API key. Ignored for Ollama (local server, unauthenticated).\n For Anthropic/Gemini: same value for both — OpenRouter key when\n OpenRouter is configured via ANTHROPIC_BASE_URL, or the direct\n provider key otherwise.\n base_url: Optional override for cloud providers' API endpoints. Testing\n hook; production leaves it None so each client uses its SDK's\n default. Ignored by OllamaClient (use ``ollama_host`` instead).\n ollama_host: Optional override for Ollama server URL. Defaults to\n ``http://localhost:11434`` (Ollama's out-of-the-box binding).\n Only used when dispatching to OllamaClient.\n\n Returns:\n A concrete AIClient subclass ready for .ask_stream() calls.\n\n Raises:\n ValueError: if model_id prefix is not recognized. Error message lists\n the supported prefixes and hints how to add a new provider.\n \"\"\"\n mid = model_id.lower()\n\n # Ollama dispatch FIRST — `llama*` and `qwen*` prefixes are unambiguous local\n # (no cloud provider ships them under those bare names).\n if mid.startswith(\"ollama/\") or any(\n mid.startswith(p) for p in _OLLAMA_BARE_PREFIXES\n ):\n return OllamaClient(\n host=ollama_host or \"http://localhost:11434\",\n model_id=model_id,\n )\n\n if mid.startswith(\"anthropic/\") or mid.startswith(\"claude\"):\n # Auto-route OpenRouter keys (sk-or-v1-*) to OpenRouter's\n # Anthropic-compat endpoint when no explicit base_url given.\n # Bundled Clicky.exe has cwd outside the repo, so .env doesn't\n # load and ANTHROPIC_BASE_URL env var is unset — without this\n # fallback the SDK defaults to api.anthropic.com and Anthropic\n # rejects the OpenRouter-namespaced key with 401 invalid x-api-key.\n # Direct Anthropic keys (sk-ant-*) leave base_url=None so the\n # SDK uses its default api.anthropic.com endpoint, where those\n # keys are valid.\n base_url = _provider_base_url(api_key, base_url, \"https://openrouter.ai/api\")\n return AnthropicClient(\n api_key=api_key, model_id=model_id, base_url=base_url,\n )\n if mid.startswith(\"google/\") or mid.startswith(\"gemini\"):\n from config import OPENROUTER_BASE_URL\n # sk-or- key → OpenRouter (GeminiClient keeps the google/ slug); a\n # direct Google AI Studio key → Google's native OpenAI-compat endpoint\n # (GeminiClient strips the google/ prefix). One OpenRouter key OR a\n # direct Google key both work.\n base_url = _provider_base_url(\n api_key, base_url, OPENROUTER_BASE_URL,\n native_url=\"https://generativelanguage.googleapis.com/v1beta/openai/\",\n )\n return GeminiClient(api_key=api_key, model_id=model_id, base_url=base_url)\n # OpenAI native (v0.3.0): 'openai/gpt-4o' etc. The GPT-Realtime\n # speech-to-speech path uses LLM_PROVIDER='openai-realtime' and is\n # handled separately in app.py (realtime.py), NOT through this factory —\n # so 'gpt-realtime*' should never reach here as a vision MODEL_ID.\n if mid.startswith(\"openai/\"):\n from config import OPENROUTER_BASE_URL\n # sk-or- key → OpenRouter (OpenAIVisionClient keeps the openai/ slug); a\n # direct OpenAI key → base_url None so the SDK uses api.openai.com (the\n # client strips the openai/ prefix + uses max_completion_tokens there).\n base_url = _provider_base_url(api_key, base_url, OPENROUTER_BASE_URL)\n return OpenAIVisionClient(\n api_key=api_key,\n model_id=model_id,\n base_url=base_url,\n )\n raise ValueError(\n f\"Unsupported MODEL_ID prefix: {model_id!r}. \"\n f\"Supported prefixes: 'anthropic/...' (or 'claude...'), \"\n f\"'google/...' (or 'gemini...'), 'openai/...', 'ollama/...' (or bare \"\n f\"'llama*'/'qwen*'/'llava*'/'mistral*'/'phi*'/'gemma*'). \"\n f\"To add a new provider, subclass AIClient in ai.py and extend \"\n f\"create_ai_client() with a new branch.\"\n )\n\n\n# --- Manual live-API verification entry point ---------------------------------\n\nif __name__ == \"__main__\":\n from config import ANTHROPIC_API_KEY, MODEL_ID\n\n print(\"=\" * 70)\n print(\"Clicky Windows -- ai.py manual live-API verification\")\n print(\" Pattern: vision-tag [POINT:x,y:label] with streaming\")\n print(\"=\" * 70)\n\n if not ANTHROPIC_API_KEY:\n raise SystemExit(\"ANTHROPIC_API_KEY missing from .env\")\n\n try:\n test_image = Image.open(\"debug_capture.jpg\").convert(\"RGB\")\n except FileNotFoundError:\n raise SystemExit(\n \"debug_capture.jpg not found in current directory. \"\n \"Run `py -3.13 -m capture` first to generate it.\"\n )\n\n client = AnthropicClient(api_key=ANTHROPIC_API_KEY, model_id=MODEL_ID)\n transcript = (\n \"what's on my screen and point at the most clickable thing \"\n \"for opening an application\"\n )\n\n print(f\"\\nSending to Claude ({MODEL_ID})...\")\n print(f\" image: {test_image.size}\")\n print(f\" transcript: {transcript!r}\")\n print(f\" max_tokens: {_CLICKY_MAX_TOKENS}\")\n\n label = f\"primary focus (image dimensions: {test_image.width}x{test_image.height} pixels)\"\n print(\"\\nStreaming response:\")\n with client.ask_stream(\n images=[(test_image, label)],\n transcript=transcript,\n history=[],\n ) as stream:\n for delta in stream.text_deltas():\n print(delta, end=\"\", flush=True)\n result = stream.final_result()\n\n print(f\"\\n\\nSpoken text (tag stripped):\")\n print(f\" {result.spoken_text!r}\")\n print(f\"\\nCoordinate: {result.coordinate}\")\n print(f\"Label: {result.element_label}\")\n print(f\"Screen: {result.screen_number}\")\n\n if result.coordinate:\n x, y = result.coordinate\n in_bounds = 0 <= x < test_image.width and 0 <= y < test_image.height\n print(f\"In bounds: {in_bounds}\")\n\n print(\"\\n\" + \"=\" * 70)\n print(\"Manual verification checklist:\")\n print(\" 1. Response text is non-empty, lowercase, casual tone\")\n print(\" 2. References specific things visible in debug_capture.jpg\")\n print(\" 3. [POINT:x,y:label] tag present at end of response\")\n print(\" 4. Coordinate is in-bounds and lands on a plausible UI element\")\n print(\" 5. Spoken text has the tag stripped (safe for TTS)\")\n print(\"=\" * 70)\n"}}}} |