1 line
63 KiB
JSON
1 line
63 KiB
JSON
{"repo": "automatebrisbaneai/voice-transcribe", "files": {"app.py": {"fd0f639507f3c0048fa0d0826d94a2bd5e7ae0b5": {"t": 1785596355.3191495, "n": 61409, "h": "93539be88f96778e", "c": "import asyncio\nimport hashlib\nimport hmac\nimport json\nimport logging\nimport os\nimport re\nimport sys\nimport time\nimport unicodedata\nimport uuid as uuid_lib\nfrom contextlib import asynccontextmanager\nfrom contextvars import ContextVar\n\nimport httpx\nfrom fastapi import FastAPI, HTTPException, Request\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom fastapi.responses import HTMLResponse\nfrom pydantic import BaseModel\nfrom pathlib import Path\n\n# ---------------------------------------------------------------------------\n# Phase 5: Structured JSON logging\n#\n# Every log line is emitted as a single JSON object to stdout.\n# Coolify's log viewer handles line-based output, and JSON is grep/jq-able.\n#\n# REQUEST_ID ContextVar carries the per-request correlation ID. Every log\n# call in the request handler picks it up automatically via the formatter.\n# ---------------------------------------------------------------------------\nREQUEST_ID: ContextVar[str] = ContextVar(\"request_id\", default=\"\")\n\n\nclass _JsonFormatter(logging.Formatter):\n \"\"\"Serialise every log record as a single JSON line.\"\"\"\n\n _SKIP_KEYS = frozenset({\n \"name\", \"msg\", \"args\", \"levelname\", \"levelno\", \"pathname\",\n \"filename\", \"module\", \"exc_info\", \"exc_text\", \"stack_info\",\n \"lineno\", \"funcName\", \"created\", \"msecs\", \"relativeCreated\",\n \"thread\", \"threadName\", \"processName\", \"process\", \"message\",\n \"taskName\",\n # uvicorn access-log records carry color_message (ANSI escape codes).\n # Uvicorn logger propagation is currently disabled, so this never\n # fires today — but skip it defensively so re-enabling propagation\n # later doesn't pollute our JSON with terminal escape sequences.\n \"color_message\",\n })\n\n def format(self, record: logging.LogRecord) -> str:\n super().format(record)\n out: dict = {\n \"timestamp\": time.strftime(\"%Y-%m-%dT%H:%M:%SZ\", time.gmtime(record.created)),\n \"level\": record.levelname,\n \"logger\": record.name,\n \"message\": record.getMessage(),\n \"request_id\": REQUEST_ID.get(\"\"),\n }\n for key, value in record.__dict__.items():\n if key not in self._SKIP_KEYS:\n out[key] = value\n if record.exc_text:\n out[\"exc\"] = record.exc_text\n return json.dumps(out, default=str)\n\n\ndef _configure_logging(level: int = logging.INFO) -> None:\n root = logging.getLogger()\n if root.handlers:\n return\n handler = logging.StreamHandler(sys.stdout)\n handler.setFormatter(_JsonFormatter())\n root.addHandler(handler)\n root.setLevel(level)\n for name in (\"uvicorn\", \"uvicorn.error\", \"uvicorn.access\"):\n logging.getLogger(name).propagate = False\n\n\n_configure_logging()\nlogger = logging.getLogger(__name__)\n\n# ---------------------------------------------------------------------------\n# Default timeouts (Phase 5 — httpx per-phase timeouts)\n# connect=5s : TCP handshake must complete quickly\n# read=30s : normal API responses (OpenCode JSON endpoints)\n# write=30s : request body upload\n# pool=5s : waiting for a free connection from the pool\n# ---------------------------------------------------------------------------\nDEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=30.0, write=30.0, pool=5.0)\n\n\n@asynccontextmanager\nasync def lifespan(application: FastAPI):\n client = httpx.AsyncClient(timeout=DEFAULT_TIMEOUT)\n application.state.http = client\n logger.info(\"Application startup\", extra={\"event\": \"app_startup\"})\n try:\n yield\n finally:\n await client.aclose()\n logger.info(\"Application shutdown\", extra={\"event\": \"app_shutdown\"})\n\n\n# docs_url/redoc_url/openapi_url all None: don't advertise the API surface\n# to anyone who guesses the conventional FastAPI paths.\napp = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)\n\nOPENCODE_GO_KEY = os.environ.get(\"OPENCODE_GO_KEY\", \"\")\n# Auto-rotation: /clean tries each key in order and skips any that has hit its\n# monthly cap (HTTP 429 / GoUsageLimitError). A single capped key took voice\n# cleanup down site-wide on 2026-06-21 because there was no fallback. Set\n# OPENCODE_GO_KEYS to a comma-separated list to enable rotation; otherwise it\n# falls back to the single OPENCODE_GO_KEY.\n_keys_env = os.environ.get(\"OPENCODE_GO_KEYS\", \"\").strip()\nOPENCODE_GO_KEYS = [k.strip() for k in _keys_env.split(\",\") if k.strip()] or (\n [OPENCODE_GO_KEY] if OPENCODE_GO_KEY else []\n)\nMODEL = \"deepseek-v4-flash\" # OpenCode Go bare slug\nOPENCODE_GO_URL = \"https://opencode.ai/zen/go/v1/chat/completions\"\n\n# Shared-secret header for /clean. Empty default keeps tests + dev unauthenticated;\n# in Coolify the env var is set and every browser pulls the value via the rendered HTML.\nCLEAN_SHARED_KEY = os.environ.get(\"CLEAN_SHARED_KEY\", \"\")\nCLEAN_SHARED_KEY_HEADER = \"X-Talk-Key\"\n\n# CORS allowlist — comma-separated origins via ALLOWED_ORIGINS env var. Adding\n# a new consumer site = update the env in Coolify + restart, no code change.\n# Default covers the three sites that consume this app today; an explicit env\n# override always wins so we don't accidentally lock new sites out on rollout.\n_DEFAULT_ORIGINS = [\n \"https://reply.croquetclaude.com\",\n \"https://talk.croquetwade.com\",\n \"https://table.croquetclaude.com\",\n]\n_origins_env = os.environ.get(\"ALLOWED_ORIGINS\", \"\").strip()\nALLOWED_ORIGINS = [\n o.strip() for o in _origins_env.split(\",\") if o.strip()\n] if _origins_env else _DEFAULT_ORIGINS\n\n# Per-IP rate limit. Default high enough that the test suite never trips it;\n# Coolify can tune via env without code changes.\nRATE_LIMIT_PER_MIN = int(os.environ.get(\"RATE_LIMIT_PER_MIN\", \"60\"))\nRATE_LIMIT_WINDOW_S = 60.0\n\nMIN_WORD_CHARS = 3\nMAX_CHUNK_CHARS = 2_000\n\n# Hard cap on the raw request body BEFORE FastAPI/Pydantic parses it.\n# MAX_CHUNK_CHARS=2000 is the application-layer limit on the cleaned text;\n# the JSON envelope adds a small fixed overhead, so 8KB gives headroom without\n# letting an attacker stream a 50MB body just to land in a 413 reply.\nMAX_REQUEST_BYTES = int(os.environ.get(\"MAX_REQUEST_BYTES\", \"8192\"))\n\n# Outline @croquetclaude mention receiver.\n# The mention is NOT answered here. We enqueue it to util PB and post an instant\n# \"working on it\" ack; the local outline-mention-watcher (on Wade's machine, where\n# Hermes + QMD live) does the answering and CC escalation. See\n# system/memory/reference_outline_mention_answering.md.\nOUTLINE_WEBHOOK_SECRET = os.environ.get(\"OUTLINE_WEBHOOK_SECRET\", \"\")\nOUTLINE_API_TOKEN = os.environ.get(\"OUTLINE_API_TOKEN\", \"\")\nOUTLINE_API_BASE = os.environ.get(\"OUTLINE_API_BASE\", \"https://docs.croquetclaude.com/api\")\n# CroquetClaude's own Outline user id — the self-filter that stops a no-@ follow-up pickup from\n# ever re-triggering on a comment WE posted (the reply-loop breaker, Feature 2.8).\nCROQUETCLAUDE_USER_ID = os.environ.get(\"CROQUETCLAUDE_USER_ID\", \"60ee9481-714c-4e65-a50b-70de0f4ff438\")\n\n# util PocketBase queue (reachable from Sydney + Wade's machine)\nUTIL_PB_URL = os.environ.get(\"UTIL_PB_URL\", \"https://util.croquetwade.com\")\nUTIL_PB_EMAIL = os.environ.get(\"UTIL_PB_EMAIL\", \"\")\nUTIL_PB_PASSWORD = os.environ.get(\"UTIL_PB_PASSWORD\", \"\")\n\n# ---------------------------------------------------------------------------\n# Per-IP rate limit — sliding window in process memory.\n#\n# Single-instance Coolify deployment: an in-process dict is sufficient.\n# If we ever scale horizontally, replace with Redis.\n# ---------------------------------------------------------------------------\nfrom collections import defaultdict, deque\n\n_rate_buckets: dict[str, deque[float]] = defaultdict(deque)\n_rate_request_count = 0 # for periodic empty-bucket purge\n\n\ndef _check_rate_limit(ip: str) -> bool:\n \"\"\"Return True if request is under the limit, False if over.\n\n Sliding-window over RATE_LIMIT_WINDOW_S. time.monotonic() is immune to\n wall-clock adjustments (NTP corrections, DST) which would otherwise let\n a buggy clock either reset or block buckets unexpectedly.\n \"\"\"\n global _rate_request_count\n now = time.monotonic()\n bucket = _rate_buckets[ip]\n cutoff = now - RATE_LIMIT_WINDOW_S\n while bucket and bucket[0] < cutoff:\n bucket.popleft()\n if len(bucket) >= RATE_LIMIT_PER_MIN:\n return False\n bucket.append(now)\n # Periodic global prune. Drop ANY bucket whose newest timestamp is older\n # than the window cutoff — covers both empty deques (handled by the same\n # condition since max() raises ValueError on empty, guarded below) and\n # one-shot IPs whose single old timestamp would otherwise leak forever.\n _rate_request_count += 1\n if _rate_request_count % 500 == 0:\n stale_keys = [\n k for k, b in _rate_buckets.items()\n if not b or b[-1] < cutoff\n ]\n for k in stale_keys:\n del _rate_buckets[k]\n return True\n\n\n_META_START_RE = re.compile(\n r\"^\\s*(sure|certainly|okay|of course|please|here is|here's|understood|\"\n r\"i (can|will|am happy|'ll clean|'ll process)|i['\\u2019]m happy)\\b\",\n re.IGNORECASE,\n)\n_SUSPICIOUS_SUBSTRINGS = (\n \"provide the transcript\", \"provide the text\", \"provide the voice\",\n \"i will process\", \"i understand\", \"i'll clean\", \"i will clean\",\n \"as an ai\", \"happy to help\",\n)\n\n# ---------------------------------------------------------------------------\n# Phase 5: Prompt-injection two-layer defence + input normalization\n# ---------------------------------------------------------------------------\n_ZERO_WIDTH_TO_SPACE = str.maketrans({\n \"\\u200b\": \" \", \"\\u200c\": \" \", \"\\u200d\": \" \",\n \"\\u200e\": \" \", \"\\u200f\": \" \", \"\\u2060\": \" \",\n \"\\ufeff\": \" \",\n})\n_PROMPT_INJECTION_KEYWORDS = (\n \"ignore previous\", \"ignore all previous\", \"disregard previous\", \"disregard instructions\",\n \"system prompt\", \"reveal instructions\", \"show instructions\", \"print instructions\",\n \"you are now\", \"new instructions:\", \"updated instructions\",\n \"[inst]\", \"<|im_start|>\", \"### system\", \"### instruction\", \"</system>\",\n \"dan mode\", \"developer mode\", \"without restrictions\", \"jailbreak\",\n \"disregard all\", \"forget all\", \"forget previous\", \"forget everything\",\n)\n\n\ndef _normalize_input(raw: str) -> str:\n # NFKC normalization handles homoglyph + width variants\n normalized = unicodedata.normalize(\"NFKC\", raw)\n # Replace zero-width chars with a space (delete would collapse word boundaries)\n normalized = normalized.translate(_ZERO_WIDTH_TO_SPACE)\n # Strip control chars except \\n and \\t\n normalized = \"\".join(c for c in normalized if c.isprintable() or c in \"\\n\\t\")\n return normalized\n\n\ndef _has_prompt_injection(text: str) -> bool:\n lower = text.lower()\n return any(kw in lower for kw in _PROMPT_INJECTION_KEYWORDS)\n\n\ndef _looks_like_meta_response(raw_input: str, model_output: str) -> bool:\n if not model_output:\n return False\n stripped = model_output.strip().lower()\n if not stripped:\n return False\n if _META_START_RE.match(stripped):\n return True\n if any(s in stripped for s in _SUSPICIOUS_SUBSTRINGS):\n return True\n # Output wildly longer than input with no shared vocabulary is suspicious.\n # Phase 5: soften for short inputs — only apply the ratio check when input\n # has at least 5 words, to avoid false positives on single-word corrections.\n in_words = {w for w in re.findall(r\"[a-z]+\", raw_input.lower()) if len(w) > 2}\n out_words = re.findall(r\"[a-z]+\", stripped)\n if len(in_words) >= 5 and len(out_words) > len(in_words) * 3:\n overlap = sum(1 for w in out_words if w in in_words)\n if overlap < max(2, len(in_words) // 3):\n return True\n return False\n\n\ndef _resolve_shared_dir() -> Path:\n \"\"\"Find the shared/ directory wherever it actually lives.\n\n Docker layout: ./shared/ is inside the app dir (Dockerfile `COPY . .` from\n the deploy submodule which has its own shared/).\n Dev layout: shared/ is a sibling at apps/shared/ (the canonical source).\n Falls back to the Docker path so `shared_file` returns clean 404s rather\n than tracebacks when neither exists.\n \"\"\"\n here = Path(__file__).parent\n for candidate in (here / \"shared\", here.parent / \"shared\"):\n if candidate.is_dir():\n return candidate\n return here / \"shared\"\n\n\nSHARED_DIR = _resolve_shared_dir()\n\n\ndef _load_croquet_dictionary() -> dict:\n \"\"\"Load croquet-dictionary.json with graceful fallback.\n\n Runs at import time, so a malformed file MUST NOT crash the boot.\n If JSON is bad (trailing comma after a manual edit, truncated write),\n log loudly and return an empty dictionary — the app still serves\n traffic, just without croquet vocab hints in the LLM prompt.\n \"\"\"\n p = SHARED_DIR / \"croquet-dictionary.json\"\n if not p.is_file():\n return {\"terms\": [], \"players\": []}\n try:\n return json.loads(p.read_text(encoding=\"utf-8\"))\n except (json.JSONDecodeError, OSError) as e:\n logger.error(\n \"croquet-dictionary.json failed to load — serving without it\",\n extra={\n \"event\": \"dictionary_load_failed\",\n \"path\": str(p),\n \"exc_type\": type(e).__name__,\n \"exc_msg\": str(e)[:200],\n },\n )\n return {\"terms\": [], \"players\": []}\n\n\n_DICTIONARY = _load_croquet_dictionary()\n_DICTIONARY_HINT = (\n \"The following are valid croquet terms that may appear in the transcript — \"\n \"correct any misheard words to match these exactly: \"\n + \", \".join(_DICTIONARY.get(\"terms\", []))\n + \". \"\n \"The following are Queensland croquet player names — \"\n \"correct any misheard names to match these exactly: \"\n + \", \".join(_DICTIONARY.get(\"players\", []))\n + \". \"\n) if (_DICTIONARY.get(\"terms\") or _DICTIONARY.get(\"players\")) else \"\"\n\n\nclass TranscriptRequest(BaseModel):\n text: str\n\n\n# ---------------------------------------------------------------------------\n# Phase 5: Request correlation ID middleware\n# ---------------------------------------------------------------------------\n@app.middleware(\"http\")\nasync def correlation_id_middleware(request: Request, call_next):\n \"\"\"\n Assign a correlation ID to every request.\n\n - Honour X-Request-ID if a reverse proxy has already set one.\n - Otherwise generate a short UUID (12 hex chars — readable in logs).\n - Store in REQUEST_ID ContextVar so every log line in this request picks\n it up automatically.\n - Return the ID in the X-Request-ID response header so clients/upstreams\n can correlate their own logs.\n \"\"\"\n request_id = request.headers.get(\"x-request-id\") or uuid_lib.uuid4().hex[:12]\n token = REQUEST_ID.set(request_id)\n # Pin on request.state too, so the global exception handler can read it\n # AFTER this middleware's finally has reset the ContextVar.\n request.state.request_id = request_id\n\n path = request.url.path\n method = request.method\n client_ip = (\n request.headers.get(\"x-forwarded-for\", \"\").split(\",\")[0].strip()\n or (request.client.host if request.client else \"unknown\")\n )\n\n t_start = time.monotonic()\n logger.info(\n \"Request start\",\n extra={\n \"event\": \"request_start\",\n \"path\": path,\n \"method\": method,\n \"client_ip\": client_ip,\n },\n )\n\n try:\n response = await call_next(request)\n duration_ms = round((time.monotonic() - t_start) * 1000)\n logger.info(\n \"Request end\",\n extra={\n \"event\": \"request_end\",\n \"path\": path,\n \"status\": response.status_code,\n \"duration_ms\": duration_ms,\n },\n )\n response.headers[\"X-Request-ID\"] = request_id\n return response\n except Exception:\n duration_ms = round((time.monotonic() - t_start) * 1000)\n logger.error(\n \"Request error\",\n extra={\n \"event\": \"request_end\",\n \"path\": path,\n \"status\": 500,\n \"duration_ms\": duration_ms,\n },\n )\n raise\n finally:\n REQUEST_ID.reset(token)\n\n\n# ---------------------------------------------------------------------------\n# Body-size guard — rejects oversized requests BEFORE FastAPI/Pydantic parses\n# the body. Two layers:\n# 1. Fast path: trust Content-Length when present. 99.9% of clients send it.\n# 2. Stream-read fallback: when Content-Length is absent (chunked transfer\n# encoding, HTTP/2 streamed requests), drain the body ourselves with a\n# running tally and 413 the moment cumulative bytes cross the cap.\n# Then rebuild the receive callable so the downstream handler still gets\n# the body it expects.\n# Without #2, any client sending `Transfer-Encoding: chunked` bypasses the cap\n# and streams unbounded bytes straight into FastAPI's Pydantic parse step.\n# Runs AFTER correlation_id middleware (registered earlier in this file).\n# ---------------------------------------------------------------------------\n@app.middleware(\"http\")\nasync def request_size_middleware(request: Request, call_next):\n from fastapi.responses import Response as _Response\n\n # Outline webhook payloads may legitimately exceed the /clean 8 KB cap.\n if request.url.path in (\"/outline/webhook\", \"/outline/webhook/\"):\n return await call_next(request)\n\n def _too_large_response(reason: str) -> _Response:\n logger.warning(\n \"Request rejected — body too large\",\n extra={\n \"event\": \"request_too_large\",\n \"reason\": reason,\n \"path\": request.url.path,\n },\n )\n return _Response(\n content=json.dumps({\"detail\": \"Payload too large\"}),\n status_code=413,\n media_type=\"application/json\",\n )\n\n # Fast path: declared Content-Length AND no chunked transfer-encoding.\n # Per RFC 7230, when both Content-Length and Transfer-Encoding are\n # present, TE wins for framing. A hostile caller can send `CL:0` plus\n # `TE:chunked` with an oversized chunked body; the CL fast path would\n # have let it through unread before this guard. When TE is present we\n # always fall through to the stream-read path so the actual bytes get\n # tallied against the cap. (Codex round-4 finding, 2026-05-24.)\n cl = request.headers.get(\"content-length\")\n te = request.headers.get(\"transfer-encoding\", \"\").lower()\n if cl is not None and \"chunked\" not in te:\n try:\n cl_int = int(cl)\n # Negative or oversize: reject. Negative CL is technically malformed\n # but uvicorn/Traefik don't always normalise; the explicit < 0 check\n # is defence-in-depth so a stray negative number can't trick the\n # `> MAX_REQUEST_BYTES` comparison into letting the body through.\n if cl_int < 0 or cl_int > MAX_REQUEST_BYTES:\n return _too_large_response(\"content_length\")\n # CL present, no chunked TE, within budget — let downstream read.\n return await call_next(request)\n except ValueError:\n # Malformed CL header — fall through to the stream-read guard\n # rather than trusting an unparseable number.\n pass\n\n # Slow path: no/unparseable Content-Length (chunked TE, HTTP/2 stream).\n # Drain the body ourselves with a running tally. As soon as we cross the\n # cap, 413 — don't keep reading more bytes than necessary to decide.\n body = bytearray()\n async for chunk in request.stream():\n body.extend(chunk)\n if len(body) > MAX_REQUEST_BYTES:\n return _too_large_response(\"streamed_overflow\")\n\n # Within budget — rebuild a receive callable that replays the body we\n # already drained, then hand off to the route handler. The route's\n # `await request.body()` (via Pydantic) will see exactly this body.\n body_bytes = bytes(body)\n consumed = False\n async def _receive():\n nonlocal consumed\n if consumed:\n return {\"type\": \"http.disconnect\"}\n consumed = True\n return {\"type\": \"http.request\", \"body\": body_bytes, \"more_body\": False}\n request._receive = _receive\n return await call_next(request)\n\n\n# ---------------------------------------------------------------------------\n# CORS — explicit origin allowlist so only the three known sites cross-fetch\n# /clean. allow_credentials=False only blocks browser-managed credentials\n# (cookies, HTTP auth); custom headers like X-Talk-Key still pass when listed\n# in allow_headers. Without X-Talk-Key in the list, browser preflight blocks\n# any cross-origin POST that tries to send it.\n#\n# Middleware registration order (Starlette/FastAPI LIFO): this is the LAST\n# middleware registered, so it wraps everything else and handles OPTIONS\n# preflight before correlation_id_middleware / request_size_middleware run.\n# ---------------------------------------------------------------------------\napp.add_middleware(\n CORSMiddleware,\n allow_origins=ALLOWED_ORIGINS,\n allow_methods=[\"GET\", \"POST\", \"OPTIONS\"],\n allow_headers=[\"Content-Type\", CLEAN_SHARED_KEY_HEADER],\n allow_credentials=False,\n max_age=86400,\n)\n\n\n# ---------------------------------------------------------------------------\n# Phase 5: Global exception handler — structured ERROR log before 500 reply\n# ---------------------------------------------------------------------------\n@app.exception_handler(Exception)\nasync def unhandled_exception_handler(request: Request, exc: Exception):\n from fastapi.responses import Response as _Response\n # By the time this runs, the middleware's `finally` has already reset the\n # REQUEST_ID ContextVar — so we read the pinned-on-state copy instead.\n request_id = getattr(request.state, \"request_id\", \"\")\n logger.error(\n \"Unhandled exception\",\n extra={\n \"event\": \"exception\",\n \"request_id\": request_id,\n \"exc_type\": type(exc).__name__,\n \"exc_msg\": str(exc),\n \"path\": request.url.path,\n },\n exc_info=True,\n )\n return _Response(\n content=json.dumps({\"detail\": \"An unexpected error occurred.\"}),\n status_code=500,\n media_type=\"application/json\",\n )\n\n\n_SYSTEM_PROMPT = (\n \"You are a transcript cleaner. Your only job: convert spoken voice into clean \"\n \"typed text without changing meaning, voice, or word choice.\\n\"\n \"\\n\"\n \"CORE RULE: If a sentence already makes sense, leave it alone. You are not \"\n \"improving the writing. You are removing fillers, adding punctuation, and fixing \"\n \"obvious transcription errors. Nothing else.\\n\"\n \"\\n\"\n \"EDIT RULES (apply in order):\\n\"\n \"\\n\"\n \"1. Self-corrections — when the speaker corrects themselves with \\\"sorry / I mean \"\n \"/ scratch that / no wait / actually / strike that\\\", keep what they corrected \"\n \"to and drop what they corrected from.\\n\"\n \"\\n\"\n \"2. Filler removal — delete: um, uh, ah, er, \\\"you know\\\" (when used as filler).\\n\"\n \"\\n\"\n \"3. Spoken commands — convert to formatting:\\n\"\n \" \\\"new line\\\" → line break\\n\"\n \" \\\"new paragraph\\\" → blank line\\n\"\n \" \\\"period\\\" → . \\\"comma\\\" → , \\\"question mark\\\" → ?\\n\"\n \"\\n\"\n \"4. Capitalisation and punctuation — add where missing.\\n\"\n \"\\n\"\n \"5. Numbers — above twelve become digits (500, 20%); 1–12 stay as words; keep \"\n \"years and proper names as-is.\\n\"\n \"\\n\"\n \"6. Spelling — British English (organise, colour, behaviour, recognise).\\n\"\n \"\\n\"\n \"7. Context fixes — repair obvious transcription errors when surrounding words \"\n \"make the intended word clear. If you cannot tell what was meant, leave the \"\n \"word alone. Examples:\\n\"\n \" \\\"the meating went well\\\" → \\\"meeting\\\"\\n\"\n \" \\\"develop a coquet strategy\\\" → \\\"croquet\\\"\\n\"\n \"\\n\"\n + ((\"KNOWN VOCABULARY — \" + _DICTIONARY_HINT + \"\\n\\n\") if _DICTIONARY_HINT else \"\")\n + \"FORBIDDEN:\\n\"\n \"- Rephrasing, reordering, or restructuring sentences.\\n\"\n \"- Adding words, transitions, headers, or commentary.\\n\"\n \"- Em dashes — use commas or full stops.\\n\"\n \"- Converting prose into bullet lists unless the speaker is clearly enumerating.\\n\"\n \"- Markdown formatting (bold, italics, headers) unless input contained it.\\n\"\n \"\\n\"\n \"OUTPUT FORMAT — read this carefully:\\n\"\n \"Output the cleaned transcript directly. No preamble like \\\"Here is\\\" or \\\"Sure\\\". \"\n \"No quotes around the output. No explanation after it. Just the cleaned text. \"\n \"If the input is empty, whitespace, a single word, or otherwise not a usable \"\n \"transcript, return it verbatim.\\n\"\n \"\\n\"\n \"EXAMPLE:\\n\"\n \"Input: um so I went to the meating yesterday sorry the meeting yesterday and we \"\n \"talked about you know the new croquet strategy for twenty twenty six\\n\"\n \"Output: I went to the meeting yesterday and we talked about the new croquet \"\n \"strategy for 2026.\"\n)\n\n\n@app.get(\"/healthz\")\nasync def healthz():\n return {\"status\": \"ok\"}\n\n\n_INDEX_PATH = Path(__file__).parent / \"index.html\"\n\n\n@app.get(\"/\")\nasync def root():\n \"\"\"Serve index.html as-is. Auth-gate bootstrap now lives inside the served\n voice-to-text.js (see shared_file), so the HTML needs no per-request rendering.\n \"\"\"\n return HTMLResponse(_INDEX_PATH.read_text(encoding=\"utf-8\"))\n\n\n# Explicit allowlist — only these two files are served from /shared/. Anything\n# else returns 404 before any filesystem probe, so URL-encoded path traversal\n# (../../etc/passwd, etc.) can't escape the bound. SHARED_DIR is resolved by\n# _resolve_shared_dir() above: tries ./shared/ (Docker layout) then ../shared/\n# (dev layout where apps/shared/ is a sibling), so both run modes serve the\n# same files without per-environment branching here.\nSHARED_FILES = {\n \"voice-to-text.js\": \"application/javascript\",\n \"croquet-dictionary.json\": \"application/json\",\n}\n\n\n@app.get(\"/shared/{filename}\")\nasync def shared_file(filename: str):\n from fastapi.responses import Response\n # Allowlist check first — anything not in the dict gets 404, no filesystem probe.\n if filename not in SHARED_FILES:\n raise HTTPException(status_code=404, detail=\"Not found\")\n content_type = SHARED_FILES[filename]\n file_path = SHARED_DIR / filename\n if not file_path.is_file():\n raise HTTPException(status_code=404, detail=\"Not found\")\n\n body = file_path.read_text(encoding=\"utf-8\")\n\n # SECURITY/UX: when serving voice-to-text.js, prepend a tiny bootstrap line\n # that sets window.VTT_EXTRA_HEADERS from the server-side CLEAN_SHARED_KEY.\n # Three wins:\n # 1. Auth gate flips via Coolify env alone — no consumer-page redeploys.\n # 2. Every consumer page (cross-domain or same-origin) gets the same\n # secret automatically — no per-app rendering, no drift.\n # 3. If CLEAN_SHARED_KEY is unset, an empty headers object is rendered\n # and the lib's whitelist filter is a no-op (dev mode).\n # The secret IS readable by anyone who view-sources the JS — that's the\n # accepted trade-off documented in the round-1 review. It still stops\n # drive-by curl/scripted abuse, which is the entire goal.\n if filename == \"voice-to-text.js\":\n bootstrap = (\n \"window.VTT_EXTRA_HEADERS=\"\n + json.dumps(\n {CLEAN_SHARED_KEY_HEADER: CLEAN_SHARED_KEY} if CLEAN_SHARED_KEY else {}\n )\n + \";\\n\"\n )\n body = bootstrap + body\n\n # CORS for public static assets: Access-Control-Allow-Origin: * so any\n # consumer page can <script src=> these. Safe because the files are\n # public static content.\n #\n # Cache-Control: no-store. The served voice-to-text.js carries the\n # X-Talk-Key bootstrap inline; if a browser caches an OLD bootstrap\n # after we rotate CLEAN_SHARED_KEY, every /clean POST from that tab\n # 403s until the cache expires. no-store eliminates that stall window\n # at the cost of one re-fetch per page load (~40KB). Trivial bandwidth,\n # zero user-visible failure on key rotation.\n headers = {\n \"Access-Control-Allow-Origin\": \"*\",\n \"Cache-Control\": \"no-store\",\n }\n return Response(\n content=body,\n media_type=content_type,\n headers=headers,\n )\n\n\n@app.post(\"/clean\")\nasync def clean_transcript(request: Request, req: TranscriptRequest):\n client: httpx.AsyncClient = request.app.state.http\n client_ip = (\n request.headers.get(\"x-forwarded-for\", \"\").split(\",\")[0].strip()\n or (request.client.host if request.client else \"unknown\")\n )\n t_start = time.monotonic()\n\n # Shared-secret gate — only active when CLEAN_SHARED_KEY env is set.\n # Generic 403 (no key name in the response body) so probing tools learn nothing.\n if CLEAN_SHARED_KEY and request.headers.get(CLEAN_SHARED_KEY_HEADER) != CLEAN_SHARED_KEY:\n logger.warning(\n \"Forbidden clean attempt (missing/invalid auth header)\",\n extra={\"event\": \"clean_forbidden\", \"client_ip\": client_ip},\n )\n raise HTTPException(status_code=403, detail=\"Forbidden\")\n\n # Per-IP rate limit — drops drive-by abuse before we hit the LLM. Trusted\n # browsers carrying the shared secret are still subject to the same limit\n # so a compromised key cannot rinse the budget from a single host.\n if not _check_rate_limit(client_ip):\n logger.warning(\n \"Rate limit exceeded\",\n extra={\"event\": \"clean_rate_limited\", \"client_ip\": client_ip},\n )\n raise HTTPException(status_code=429, detail=\"Too many requests\")\n\n raw = req.text or \"\"\n\n # Phase 5: normalize input — NFKC, strip zero-width chars, strip control chars.\n normalized = _normalize_input(raw)\n\n # Phase 5: chunk size cap — reject oversized inputs before touching the LLM.\n if len(normalized) > MAX_CHUNK_CHARS:\n raise HTTPException(status_code=413, detail=\"Chunk too large.\")\n\n # Phase 5: prompt-injection guard — short-circuit without calling the LLM.\n if _has_prompt_injection(normalized):\n logger.warning(\n \"Prompt injection detected — short-circuiting LLM\",\n extra={\"event\": \"prompt_injection_detected\", \"input_preview\": normalized[:120]},\n )\n return {\"cleaned\": raw} # return ORIGINAL raw, not normalized — preserve user intent\n\n # Input guard — too few word chars means there's nothing to clean.\n word_chars = sum(1 for c in normalized if c.isalpha())\n if word_chars < MIN_WORD_CHARS:\n return {\"cleaned\": raw}\n\n input_chars = len(normalized)\n\n try:\n t_or_start = time.monotonic()\n payload = {\n \"model\": MODEL,\n \"messages\": [\n {\"role\": \"system\", \"content\": _SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": normalized},\n ],\n # DeepSeek V4 on OpenCode burns substantial hidden reasoning tokens\n # against max_tokens (OR's reasoning={\"effort\":\"none\"} extension is\n # silently ignored). 500-token floor was too tight — reasoning ate\n # every token, content came back empty. 4096 covers reasoning + actual\n # cleaned output even for the longest practical transcripts.\n \"max_tokens\": 4096,\n }\n\n # Auto-rotation across OPENCODE_GO_KEYS: try each key, skip any that is\n # capped (429 / GoUsageLimitError) or errors, fall through to the next.\n data = None\n res = None\n last_status = None\n last_body = None\n for key_idx, key in enumerate(OPENCODE_GO_KEYS):\n res = await client.post(\n OPENCODE_GO_URL,\n headers={\n \"Authorization\": f\"Bearer {key}\",\n \"Content-Type\": \"application/json\",\n \"User-Agent\": \"croquetwade-worker/1.0\",\n },\n json=payload,\n )\n last_status = res.status_code\n try:\n body = res.json()\n except ValueError:\n body = {}\n if res.status_code == 200 and isinstance(body, dict) and \"choices\" in body:\n data = body\n if key_idx > 0:\n logger.info(\n \"OpenCode key rotated to a working key\",\n extra={\"event\": \"clean_key_rotated\", \"key_index\": key_idx},\n )\n break\n capped = res.status_code == 429 or \"limit\" in str(body).lower()\n logger.warning(\n \"OpenCode key unavailable — rotating to next\",\n extra={\n \"event\": \"clean_key_skip\",\n \"key_index\": key_idx,\n \"status_code\": res.status_code,\n \"capped\": capped,\n \"client_ip\": client_ip,\n },\n )\n last_body = body\n or_duration_ms = round((time.monotonic() - t_or_start) * 1000)\n\n if data is None:\n logger.error(\n \"Upstream LLM error — all OpenCode keys exhausted\",\n extra={\n \"event\": \"upstream_clean\",\n \"status_code\": last_status,\n \"duration_ms\": or_duration_ms,\n \"input_chars\": input_chars,\n \"output_chars\": 0,\n \"error\": str(last_body),\n \"client_ip\": client_ip,\n },\n )\n duration_ms = round((time.monotonic() - t_start) * 1000)\n logger.warning(\n \"Clean failed — upstream_error\",\n extra={\n \"event\": \"clean_failure\",\n \"duration_ms\": duration_ms,\n \"failure_reason\": \"upstream_error_all_keys\",\n },\n )\n raise HTTPException(status_code=502, detail=\"Transcript cleaning failed, please try again.\")\n\n cleaned = data[\"choices\"][0][\"message\"][\"content\"]\n output_chars = len(cleaned)\n or_duration_ms = round((time.monotonic() - t_or_start) * 1000)\n\n logger.info(\n \"Upstream clean succeeded\",\n extra={\n \"event\": \"upstream_clean\",\n \"status_code\": res.status_code,\n \"duration_ms\": or_duration_ms,\n \"input_chars\": input_chars,\n \"output_chars\": output_chars,\n },\n )\n\n if _looks_like_meta_response(normalized, cleaned):\n logger.warning(\n \"Suspicious model output — falling back to raw input\",\n extra={\n \"event\": \"clean_suspicious_output\",\n \"input_chars\": input_chars,\n \"output_chars\": output_chars,\n \"input_preview\": normalized[:120],\n \"output_preview\": cleaned[:240],\n },\n )\n cleaned = raw\n output_chars = len(cleaned)\n\n duration_ms = round((time.monotonic() - t_start) * 1000)\n logger.info(\n \"Clean succeeded\",\n extra={\n \"event\": \"clean_request\",\n \"duration_ms\": duration_ms,\n \"input_chars\": input_chars,\n \"output_chars\": output_chars,\n \"client_ip\": client_ip,\n },\n )\n return {\"cleaned\": cleaned}\n\n except HTTPException:\n raise\n except Exception as e:\n duration_ms = round((time.monotonic() - t_start) * 1000)\n logger.error(\n \"Clean unexpected error\",\n extra={\n \"event\": \"clean_failure\",\n \"duration_ms\": duration_ms,\n \"failure_reason\": \"internal_error\",\n \"exc_type\": type(e).__name__,\n \"exc_msg\": str(e),\n },\n )\n raise HTTPException(status_code=502, detail=\"Transcript cleaning failed, please try again.\")\n\n\n# ---------------------------------------------------------------------------\n# Outline @croquetclaude mention receiver\n#\n# Outline fires a webhook for every comments.create and documents.update event.\n# This route extracts ProseMirror mention nodes (not Markdown, which omits them),\n# checks for @croquetclaude, and replies as CroquetClaude via the Outline API.\n#\n# Required Coolify env vars on this service:\n# OUTLINE_API_TOKEN — CroquetClaude's member token\n# OPENROUTER_KEY — OpenRouter key for the answer LLM\n# OUTLINE_WEBHOOK_SECRET — Outline webhook signing secret (set in Outline → Integrations → Webhooks)\n# ---------------------------------------------------------------------------\n\n_MENTION_RE_OL = re.compile(r\"(?<!\\w)@croquetclaude\\b\", re.I)\n# A native @mention NODE (what the autocomplete inserts) is NOT literal \"@croquetclaude\" text. In the\n# markdown export it is @[Label](mention://<workspace>/user/<userId>), and documents.info returns this\n# markdown `text` (NOT the ProseMirror `data`), so a body @mention is detected by CroquetClaude's user\n# id appearing in a mention:// URL. (Comment payloads do carry `data`, so the comment path is fine.)\n_MENTION_NODE_RE = re.compile(r\"mention://[^\\s)]*?/user/\" + re.escape(CROQUETCLAUDE_USER_ID), re.I)\n_MENTION_MD_RE = re.compile(r\"@\\[([^\\]]+)\\]\\(mention://[^)]+\\)\") # @[Name](mention://...) -> @Name\n_webhook_seen: set[str] = set() # in-process fast dedup; the durable dedup is the util PB unique comment_id\n\n# Debounce for document-body mentions (live incident 2026-06-11): Outline AUTOSAVES every few\n# seconds while someone types, and every save is a documents.update event carrying a longer\n# half-typed snapshot of the sentence — four acks and an answer to \"can you get d\" before the ask\n# was even finished. Each new save for a doc resets its timer; only DOC_MENTION_DEBOUNCE_S after\n# the LAST save do we read the doc fresh and enqueue the settled line. In-memory (a restart\n# mid-window drops that one timer — the member just edits the line again).\nDOC_MENTION_DEBOUNCE_S = int(os.environ.get(\"DOC_MENTION_DEBOUNCE_S\", \"60\"))\n_doc_debounce: dict[str, \"asyncio.Task\"] = {}\n\n\ndef _is_cc_mention(s: str) -> bool:\n \"\"\"True if the text mentions CroquetClaude either as literal @croquetclaude or a native @mention\n node (markdown @[..](mention://.../user/<cc id>)).\"\"\"\n return bool(_MENTION_RE_OL.search(s or \"\") or _MENTION_NODE_RE.search(s or \"\"))\n\n\ndef _clean_mention_md(text: str) -> str:\n \"\"\"Turn a markdown @mention node @[Label](mention://...) into a plain @Label for the queued text.\"\"\"\n return _MENTION_MD_RE.sub(r\"@\\1\", text or \"\")\n\n# Canned instant acknowledgement (no LLM). CroquetClaude voice, AU English, no em-dashes.\n_ACK_TEXT = (\n \"⏳ Got your message — it's in the queue. I'll bring the answer back right here.\"\n)\n\n\ndef _pm_node_to_text(node) -> str:\n \"\"\"Recursively extract plain text from a ProseMirror node.\n\n Native Outline @mention creates a structured node (type=mention, attrs.label=Name).\n These are invisible in the Markdown text export — the ProseMirror data field is required.\n \"\"\"\n if not isinstance(node, dict):\n return \"\"\n out = []\n for child in (node.get(\"content\") or []):\n t = child.get(\"type\")\n if t == \"text\":\n out.append(child.get(\"text\", \"\"))\n elif t == \"mention\":\n label = (child.get(\"attrs\") or {}).get(\"label\", \"\")\n if label:\n out.append(f\"@{label}\")\n elif child.get(\"content\"):\n sub = _pm_node_to_text(child)\n if sub.strip():\n out.append(sub)\n out.append(\"\\n\")\n return \"\".join(out)\n\n\nasync def _outline_api(client: httpx.AsyncClient, action: str, payload: dict) -> dict:\n r = await client.post(\n f\"{OUTLINE_API_BASE}/{action}\",\n headers={\n \"Authorization\": f\"Bearer {OUTLINE_API_TOKEN}\",\n \"Content-Type\": \"application/json\",\n },\n json=payload,\n timeout=httpx.Timeout(connect=5.0, read=20.0, write=10.0, pool=5.0),\n )\n return r.json()\n\n\n# --- util PB queue helpers (the mention is answered by the local worker, not here) ---\n\n_utilpb = {\"token\": None, \"exp\": 0.0}\n\n\nasync def _utilpb_token(client: httpx.AsyncClient) -> str:\n now = time.time()\n if _utilpb[\"token\"] and now < _utilpb[\"exp\"]:\n return _utilpb[\"token\"]\n r = await client.post(\n f\"{UTIL_PB_URL}/api/collections/_superusers/auth-with-password\",\n json={\"identity\": UTIL_PB_EMAIL, \"password\": UTIL_PB_PASSWORD},\n timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=5.0),\n )\n r.raise_for_status()\n tok = r.json()[\"token\"]\n _utilpb[\"token\"] = tok\n _utilpb[\"exp\"] = now + 25 * 60\n return tok\n\n\nasync def _enqueue_mention(client: httpx.AsyncClient, **fields) -> str:\n \"\"\"Insert a queue row. Returns the new row id, or \"exists\" if this comment_id is\n already queued (durable dedup via the unique index). Raises on any other failure.\"\"\"\n token = await _utilpb_token(client)\n rec = {\"status\": \"new\", \"attempt_count\": 0, **fields}\n r = await client.post(\n f\"{UTIL_PB_URL}/api/collections/outline_mentions/records\",\n json=rec, headers={\"Authorization\": token},\n timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=5.0),\n )\n if r.status_code in (200, 201):\n return r.json()[\"id\"]\n # The unique partial index on comment_id reports this exact code on a duplicate.\n if r.status_code == 400 and \"validation_not_unique\" in r.text:\n return \"exists\"\n r.raise_for_status()\n raise RuntimeError(f\"enqueue failed: {r.status_code} {r.text[:200]}\")\n\n\nasync def _patch_row(client: httpx.AsyncClient, row_id: str, fields: dict) -> None:\n token = await _utilpb_token(client)\n r = await client.patch(\n f\"{UTIL_PB_URL}/api/collections/outline_mentions/records/{row_id}\",\n json=fields, headers={\"Authorization\": token},\n timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=5.0),\n )\n r.raise_for_status() # surface a failed patch (e.g. lost ack_comment_id) to the caller's logger\n\n\nasync def _post_ack(client: httpx.AsyncClient, doc_id: str, parent_id: str) -> str:\n \"\"\"Post the instant 'working on it' ack as CroquetClaude. Returns its comment id.\"\"\"\n payload = {\"documentId\": doc_id, \"text\": _ACK_TEXT}\n if parent_id:\n payload[\"parentCommentId\"] = parent_id\n resp = await _outline_api(client, \"comments.create\", payload)\n return ((resp.get(\"data\") or {}).get(\"id\")) or \"\"\n\n\nasync def _ack_and_patch(client: httpx.AsyncClient, row_id: str, doc_id: str, parent_id: str) -> None:\n \"\"\"Post the ack and record its id on the row. An ack failure is non-fatal — the row is\n already queued, so the worker will still answer (threaded under the mention's parent).\"\"\"\n try:\n ack_id = await _post_ack(client, doc_id, parent_id)\n if ack_id:\n await _patch_row(client, row_id, {\"ack_comment_id\": ack_id})\n except Exception as exc: # noqa: BLE001\n logger.warning(\"Outline ack post failed (row queued anyway)\",\n extra={\"event\": \"outline_ack_failed\", \"row\": row_id, \"exc\": str(exc)})\n\n\nasync def _find_row_by_parent(client: httpx.AsyncClient, parent_id: str) -> dict | None:\n \"\"\"Find the queue row for the thread a reply belongs to (Feature 2.8 follow-ups).\n\n Outline threads are flat: a reply's parentCommentId is the THREAD ROOT (the original\n @mention comment), not the specific CroquetClaude comment it sits under. So a reply's\n parent matches the original mention row's `comment_id`. We also match `ack_comment_id`\n / `answer_comment_id` defensively (in case a client ever parents to CC's comment directly,\n and because answer_comment_id is not always written back).\"\"\"\n if not parent_id:\n return None\n token = await _utilpb_token(client)\n flt = (f'(comment_id=\"{parent_id}\" || ack_comment_id=\"{parent_id}\" '\n f'|| answer_comment_id=\"{parent_id}\")')\n r = await client.get(\n f\"{UTIL_PB_URL}/api/collections/outline_mentions/records\",\n params={\"filter\": flt, \"perPage\": 1},\n headers={\"Authorization\": token},\n timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=5.0),\n )\n if r.status_code != 200:\n return None\n items = (r.json() or {}).get(\"items\") or []\n return items[0] if items else None\n\n\nasync def _cc_was_last_in_thread(client: httpx.AsyncClient, doc_id: str,\n thread_root: str, this_comment_id: str) -> bool:\n \"\"\"True if CroquetClaude posted the most recent comment in this thread before `this_comment_id`.\n\n This is the \"don't be rude\" guard: we only answer a no-@ reply when it is genuinely a reply\n TO CroquetClaude (CC spoke last), not when committee members are talking among themselves in\n a thread that merely started with a question to CC. Fails safe to False on any error.\"\"\"\n if not doc_id or not thread_root:\n return False\n try:\n resp = await _outline_api(client, \"comments.list\", {\"documentId\": doc_id, \"limit\": 100})\n except Exception:\n return False\n comments = resp.get(\"data\") or []\n # The thread = the root comment itself plus everything parented to it (flat threading).\n thread = [c for c in comments\n if c.get(\"id\") == thread_root or (c.get(\"parentCommentId\") or \"\") == thread_root]\n # Exclude the triggering reply — we want who spoke last BEFORE it.\n thread = [c for c in thread if c.get(\"id\") != this_comment_id]\n if not thread:\n return False\n thread.sort(key=lambda c: c.get(\"createdAt\") or \"\", reverse=True)\n last = thread[0]\n author_id = (last.get(\"createdBy\") or {}).get(\"id\") or last.get(\"createdById\") or \"\"\n return author_id == CROQUETCLAUDE_USER_ID\n\n\nasync def _handle_followup(client: httpx.AsyncClient, model: dict) -> bool:\n \"\"\"A no-@ reply in a thread CroquetClaude is in -> enqueue it as a follow-up + ack, but only\n when CroquetClaude was the last to speak (a genuine reply to it). Returns True if handled as a\n follow-up, False if it's just a normal committee comment. The caller has already applied the\n self-filter, so this is never one of our own comments.\"\"\"\n parent_id = model.get(\"parentCommentId\") or \"\"\n row = await _find_row_by_parent(client, parent_id)\n if not row:\n return False\n comment_id = model.get(\"id\", \"\")\n doc_id = model.get(\"documentId\", \"\") or row.get(\"doc_id\", \"\")\n # Only chime in when CC posted the most recent comment in the thread — i.e. this is a reply TO\n # CroquetClaude, not committee members chatting. Without this, every reply in the thread answers.\n if not await _cc_was_last_in_thread(client, doc_id, parent_id, comment_id):\n return False\n asker = (model.get(\"createdBy\") or {}).get(\"name\", \"a committee member\")\n text = _pm_node_to_text(model.get(\"data\") or {})\n res = await _enqueue_mention(\n client, comment_id=comment_id, doc_id=doc_id, source=\"comment\",\n parent_comment_id=parent_id, mention_text=text, asker=asker,\n doc_title=row.get(\"doc_title\", \"\"), doc_url=row.get(\"doc_url\", \"\"),\n )\n if res == \"exists\":\n return True # already queued — no second ack\n logger.info(\"Outline follow-up (no @) queued\",\n extra={\"event\": \"outline_followup_queued\", \"comment_id\": comment_id, \"row\": res})\n await _ack_and_patch(client, res, doc_id, parent_id)\n return True\n\n\nasync def _handle_comment_mention(client: httpx.AsyncClient, data: dict) -> None:\n \"\"\"Enqueue a comment @mention for the local worker, then post the instant ack.\"\"\"\n comment_id = data.get(\"id\", \"\")\n doc_id = data.get(\"documentId\", \"\")\n parent_id = data.get(\"parentCommentId\") or comment_id # thread root\n asker = (data.get(\"createdBy\") or {}).get(\"name\", \"a committee member\")\n text = _pm_node_to_text(data.get(\"data\") or {})\n\n res = await _enqueue_mention(\n client, comment_id=comment_id, doc_id=doc_id, source=\"comment\",\n parent_comment_id=parent_id, mention_text=text, asker=asker,\n )\n if res == \"exists\":\n return # already queued — no second ack\n logger.info(\"Outline comment mention queued\",\n extra={\"event\": \"outline_mention_queued\", \"comment_id\": comment_id, \"row\": res})\n await _ack_and_patch(client, res, doc_id, parent_id)\n\n\nasync def _handle_doc_mention(client: httpx.AsyncClient, doc_id: str, asker: str = \"\") -> None:\n \"\"\"Enqueue a document-body @mention, reading the doc FRESH (the debounce means the sentence has\n settled by now). Body detection is NODE-ONLY (`mention://.../user/<cc id>` — what the\n autocomplete inserts): literal \"@croquetclaude\" text in a page body no longer triggers, because\n our own guide and tour pages quote that text as examples and answered themselves when created\n (2026-06-11). Comments keep the literal-text trigger.\"\"\"\n if not doc_id:\n return\n resp = await _outline_api(client, \"documents.info\", {\"id\": doc_id})\n doc = resp.get(\"data\") or {}\n body_text = doc.get(\"text\") or _pm_node_to_text(doc.get(\"data\") or {})\n if not _MENTION_NODE_RE.search(body_text):\n return\n lines = [ln.strip() for ln in body_text.splitlines() if ln.strip() and _MENTION_NODE_RE.search(ln)]\n if not lines:\n return\n line = _clean_mention_md(lines[0])\n # Synthetic comment_id so the unique-index dedup covers body mentions too.\n comment_id = f\"body:{doc_id}:{hashlib.sha1(line.encode()).hexdigest()[:16]}\"\n if comment_id in _webhook_seen:\n return\n _webhook_seen.add(comment_id)\n\n doc_url = f\"https://docs.croquetclaude.com{doc.get('url', '')}\" if doc.get(\"url\") else \"\"\n try:\n res = await _enqueue_mention(\n client, comment_id=comment_id, doc_id=doc_id, source=\"document\",\n parent_comment_id=\"\", mention_text=line, asker=asker or \"a committee member\",\n doc_title=doc.get(\"title\", \"\"), doc_url=doc_url,\n )\n except Exception:\n _webhook_seen.discard(comment_id) # let an Outline retry re-run (mirror the comment path)\n raise\n if res == \"exists\":\n return\n logger.info(\"Outline doc mention queued\",\n extra={\"event\": \"outline_doc_mention_queued\", \"doc_id\": doc_id, \"row\": res})\n await _ack_and_patch(client, res, doc_id, parent_id=\"\") # top-level ack for body mentions\n\n\nasync def _debounced_doc_mention(client: httpx.AsyncClient, doc_id: str, actor_id: str) -> None:\n \"\"\"Wait out the autosave storm, then resolve the actor's name (asker register) and handle the\n settled mention. Cancellation = a newer save superseded this timer; the new task owns the doc.\"\"\"\n try:\n await asyncio.sleep(DOC_MENTION_DEBOUNCE_S)\n asker = \"\"\n if actor_id:\n try:\n u = await _outline_api(client, \"users.info\", {\"id\": actor_id})\n asker = (u.get(\"data\") or {}).get(\"name\") or \"\"\n except Exception: # noqa: BLE001 — the name is a nicety, never block the mention\n asker = \"\"\n await _handle_doc_mention(client, doc_id, asker=asker)\n except asyncio.CancelledError:\n pass\n except Exception as exc: # noqa: BLE001 — log like the old inline path did\n logger.error(\"Outline doc webhook error (debounced)\",\n extra={\"event\": \"outline_webhook_error\", \"exc\": str(exc)})\n finally:\n if _doc_debounce.get(doc_id) is asyncio.current_task():\n _doc_debounce.pop(doc_id, None)\n\n\ndef _schedule_doc_mention(client: httpx.AsyncClient, doc_id: str, actor_id: str) -> None:\n \"\"\"Debounce documents.update per doc: each new save cancels the pending timer and starts a\n fresh one, so the mention is enqueued exactly once, DOC_MENTION_DEBOUNCE_S after typing stops.\"\"\"\n old = _doc_debounce.get(doc_id)\n if old and not old.done():\n old.cancel()\n _doc_debounce[doc_id] = asyncio.create_task(_debounced_doc_mention(client, doc_id, actor_id))\n\n\nasync def _handle_collection_mention(client: httpx.AsyncClient, col_id: str, asker: str = \"\") -> None:\n \"\"\"A mention typed into a COLLECTION's cover text (its description). Found live 2026-06-12:\n Wade asked there and nothing answered — covers fire collections.update, which we'd never\n subscribed to, and they have no comment threads of their own. The answer surface is the\n collection's first page (its hub), with the cover ask quoted so the thread makes sense.\"\"\"\n if not col_id:\n return\n info = await _outline_api(client, \"collections.info\", {\"id\": col_id})\n col = info.get(\"data\") or {}\n desc = col.get(\"description\") or \"\"\n if not _MENTION_NODE_RE.search(desc):\n return\n lines = [ln.strip() for ln in desc.splitlines() if ln.strip() and _MENTION_NODE_RE.search(ln)]\n if not lines:\n return\n line = _clean_mention_md(lines[0])\n comment_id = f\"coldesc:{col_id}:{hashlib.sha1(line.encode()).hexdigest()[:16]}\"\n if comment_id in _webhook_seen:\n return\n _webhook_seen.add(comment_id)\n\n # anchor the conversation on the collection's first page (covers can't hold comments)\n try:\n tree = await _outline_api(client, \"collections.documents\", {\"id\": col_id})\n anchor = ((tree.get(\"data\") or [{}])[0]) or {}\n except Exception:\n anchor = {}\n anchor_id = anchor.get(\"id\") or \"\"\n if not anchor_id:\n logger.warning(\"collection cover mention but the collection has no pages to answer on\",\n extra={\"event\": \"outline_coldesc_no_anchor\", \"collection\": col_id})\n _webhook_seen.discard(comment_id)\n return\n mention_text = f\"[asked on the '{col.get('name', 'collection')}' collection cover] {line}\"\n try:\n res = await _enqueue_mention(\n client, comment_id=comment_id, doc_id=anchor_id, source=\"document\",\n parent_comment_id=\"\", mention_text=mention_text, asker=asker or \"a committee member\",\n doc_title=anchor.get(\"title\", \"\"), doc_url=\"\",\n )\n except Exception:\n _webhook_seen.discard(comment_id)\n raise\n if res == \"exists\":\n return\n logger.info(\"Outline collection-cover mention queued\",\n extra={\"event\": \"outline_coldesc_mention_queued\", \"collection\": col_id, \"row\": res})\n await _ack_and_patch(client, res, anchor_id, parent_id=\"\")\n\n\nasync def _debounced_collection_mention(client: httpx.AsyncClient, col_id: str, actor_id: str) -> None:\n \"\"\"Same debounce shape as documents: wait out the autosave storm, resolve the asker, handle.\"\"\"\n try:\n await asyncio.sleep(DOC_MENTION_DEBOUNCE_S)\n asker = \"\"\n if actor_id:\n try:\n u = await _outline_api(client, \"users.info\", {\"id\": actor_id})\n asker = (u.get(\"data\") or {}).get(\"name\") or \"\"\n except Exception: # noqa: BLE001\n asker = \"\"\n await _handle_collection_mention(client, col_id, asker=asker)\n except asyncio.CancelledError:\n pass\n except Exception as exc: # noqa: BLE001\n logger.error(\"Outline collection webhook error (debounced)\",\n extra={\"event\": \"outline_webhook_error\", \"exc\": str(exc)})\n finally:\n if _doc_debounce.get(f\"col:{col_id}\") is asyncio.current_task():\n _doc_debounce.pop(f\"col:{col_id}\", None)\n\n\ndef _schedule_collection_mention(client: httpx.AsyncClient, col_id: str, actor_id: str) -> None:\n key = f\"col:{col_id}\"\n old = _doc_debounce.get(key)\n if old and not old.done():\n old.cancel()\n _doc_debounce[key] = asyncio.create_task(_debounced_collection_mention(client, col_id, actor_id))\n\n\ndef _verify_outline_sig(body: bytes, headers) -> bool:\n \"\"\"Verify an Outline webhook signature.\n\n Outline signs Stripe-style: header `Outline-Signature: t=<ts>,s=<hmac>` where\n hmac = HMAC-SHA256(secret, \"<ts>.<raw-body>\"). We also accept the legacy\n `X-Outline-Signature: sha256=<hmac(body)>` scheme as a fallback, and try a\n body-only variant, so a minor format difference can't lock us out.\n \"\"\"\n secret = OUTLINE_WEBHOOK_SECRET.encode()\n ol = headers.get(\"outline-signature\", \"\")\n if ol and \"s=\" in ol:\n parts = {}\n for kv in ol.split(\",\"):\n k, _, v = kv.partition(\"=\")\n parts[k.strip()] = v.strip()\n t, s = parts.get(\"t\", \"\"), parts.get(\"s\", \"\")\n if s:\n for msg in (t.encode() + b\".\" + body, body):\n if hmac.compare_digest(s, hmac.new(secret, msg, hashlib.sha256).hexdigest()):\n return True\n x = headers.get(\"x-outline-signature\", \"\")\n if x:\n if hmac.compare_digest(x, \"sha256=\" + hmac.new(secret, body, hashlib.sha256).hexdigest()):\n return True\n return False\n\n\n@app.post(\"/outline/webhook\")\n@app.post(\"/outline/webhook/\")\nasync def outline_webhook(request: Request):\n \"\"\"Receive Outline webhook events, detect @croquetclaude mentions, reply as CroquetClaude.\"\"\"\n body = await request.body()\n\n # HMAC signature verification — skip if OUTLINE_WEBHOOK_SECRET is not configured.\n if OUTLINE_WEBHOOK_SECRET:\n if not _verify_outline_sig(body, request.headers):\n ol = request.headers.get(\"outline-signature\", \"\")\n xol = request.headers.get(\"x-outline-signature\", \"\")\n logger.warning(\n \"Outline webhook bad signature\",\n extra={\"event\": \"outline_webhook_bad_sig\",\n \"stripe_style\": bool(ol and \"s=\" in ol),\n \"legacy_style\": bool(xol),\n \"sig_preview\": (ol or xol)[:28]},\n )\n raise HTTPException(status_code=401, detail=\"Bad signature\")\n\n try:\n payload = json.loads(body)\n except (json.JSONDecodeError, ValueError):\n raise HTTPException(status_code=400, detail=\"Bad JSON\")\n\n event = payload.get(\"event\", \"\")\n # Outline wraps the entity as payload.payload.model (NOT payload.data).\n model = (payload.get(\"payload\") or {}).get(\"model\") or {}\n client: httpx.AsyncClient = request.app.state.http\n\n if event == \"comments.create\":\n comment_id = model.get(\"id\", \"\")\n if not comment_id or comment_id in _webhook_seen:\n return {\"ok\": True}\n # Self-filter FIRST, for BOTH branches: never act on a comment CroquetClaude posted. This is\n # the reply-loop breaker — without it, an answer that merely quotes \"@croquetclaude\" would\n # re-enqueue itself via the @-branch and the bot would answer its own answer.\n author_id = (model.get(\"createdBy\") or {}).get(\"id\") or model.get(\"createdById\") or \"\"\n if author_id and author_id == CROQUETCLAUDE_USER_ID:\n return {\"ok\": True}\n text = _pm_node_to_text(model.get(\"data\") or {})\n if _MENTION_RE_OL.search(text):\n _webhook_seen.add(comment_id)\n try:\n await _handle_comment_mention(client, model)\n except Exception as exc:\n # Enqueue failed (e.g. util PB unreachable). By design we post NO ack we\n # can't fulfil; drop the in-process flag so an Outline retry can succeed.\n logger.error(\n \"Outline comment webhook error\",\n extra={\"event\": \"outline_webhook_error\", \"exc\": str(exc)},\n )\n _webhook_seen.discard(comment_id)\n else:\n # No @ — people forget to @ when replying in a thread. If this is a direct reply to a\n # CroquetClaude comment, pick it up as a follow-up (Feature 2.8). The CC self-filter is\n # already applied above; here we additionally require a KNOWN author and fail safe if it\n # is absent (better to miss a follow-up than risk acting on an unattributable comment).\n if not author_id:\n return {\"ok\": True}\n _webhook_seen.add(comment_id)\n try:\n if not await _handle_followup(client, model):\n _webhook_seen.discard(comment_id) # not a follow-up; a normal committee comment\n except Exception as exc:\n logger.error(\n \"Outline follow-up webhook error\",\n extra={\"event\": \"outline_webhook_error\", \"exc\": str(exc)},\n )\n _webhook_seen.discard(comment_id)\n\n elif event == \"documents.update\":\n doc_id = model.get(\"id\", \"\")\n actor_id = payload.get(\"actorId\") or \"\"\n # CroquetClaude's own page writes (proposal blocks, accept/subpage swaps, doc updates via\n # the API) must never trigger the mention path — the tour pages answered their own example\n # text when created (2026-06-11). Humans (or an absent actorId) proceed to the debounce.\n if doc_id and actor_id != CROQUETCLAUDE_USER_ID:\n _schedule_doc_mention(client, doc_id, actor_id)\n\n elif event == \"collections.update\":\n col_id = model.get(\"id\", \"\")\n actor_id = payload.get(\"actorId\") or \"\"\n if col_id and actor_id != CROQUETCLAUDE_USER_ID:\n _schedule_collection_mention(client, col_id, actor_id)\n\n return {\"ok\": True}\n"}}}} |