1 line
87 KiB
JSON
1 line
87 KiB
JSON
{"repo": "1314520alu/gcs", "files": {"translate_params_ai.py": {"5c89cd04a7741c23d5d8bff715ac05edbe4bc3d9": {"t": 1785746405.9964695, "n": 35813, "h": "ccb6c0a548af1f61", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nTranslate ArduPilot parameter metadata using OpenAI (Responses API) or xAI (Chat Completions).\n\nThis script reads English source strings from JS/data/apm-param-db.en.json,\ntranslates them (primarily the long 'd' description field) into Simplified Chinese\nin batches, caches results locally, and writes the merged output back to\nJS/data/apm-param-db.json.\n\nSupported providers:\n - OpenAI (default for keys not starting with \"xai-\")\n - xAI (Grok) — auto-detected when the API key starts with \"xai-\",\n or explicitly selected with --provider xai. Recommended default: grok-4.3\n - DeepSeek — explicitly selected with --provider deepseek.\n Recommended default: deepseek-v4-flash (user-confirmed working model)\n\nEnvironment variables (recommended):\n OPENAI_API_KEY For OpenAI\n XAI_API_KEY For xAI (Grok)\n DEEPSEEK_API_KEY For DeepSeek\n\nExample (xAI):\n export XAI_API_KEY=\"xai-...\"\n python translate_params_ai.py --review --limit 300 --api-key-env XAI_API_KEY\n\nExample (DeepSeek):\n export DEEPSEEK_API_KEY=\"sk-...\"\n python translate_params_ai.py --review --limit 300 \\\n --api-key-env DEEPSEEK_API_KEY --provider deepseek --model deepseek-v4-flash\n\nExample (OpenAI):\n export OPENAI_API_KEY=\"sk-...\"\n python translate_params_ai.py --model gpt-5.4-mini --limit 200\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport os\nimport shutil\nimport sys\nimport time\nimport urllib.error\nimport urllib.request\nfrom pathlib import Path\nfrom typing import Any\n\nROOT = Path(__file__).resolve().parent\nDB_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.json\"\nSOURCE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.en.json\"\nLEGACY_CACHE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-translate-cache.json\"\nAI_CACHE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-translate-cache.ai.json\"\nBACKUP_OUT_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.pre-ai-backup.json\"\n\n# Reviewable workflow artifacts (仅 d 字段)\nREVIEW_DIR = ROOT / \"tools\" / \"translation-review\"\nGLOSSARY_PATH = ROOT / \"tools\" / \"glossary-param-translation.json\"\n\n# Provider-aware defaults\nOPENAI_DEFAULT_MODEL = \"gpt-5.4-mini\"\nOPENAI_API_URL = \"https://api.openai.com/v1/responses\"\n\n# Current recommended default for xAI (as of late May 2026).\n# grok-2 / grok-3 and early grok-4 variants were retired on 2026-05-15.\n# grok-4.3 is the current flagship (best quality/speed for technical translation).\n# For heavier reasoning on complex descriptions you may override with:\n# --model grok-4.20-0309-reasoning\nXAI_DEFAULT_MODEL = \"grok-4.3\"\nXAI_API_URL = \"https://api.x.ai/v1/chat/completions\"\n\n# DeepSeek (OpenAI-compatible Chat Completions + json_schema support)\n# User confirmed \"deepseek-v4-flash\" works for their use case.\n# Standard base for DeepSeek is https://api.deepseek.com\nDEEPSEEK_DEFAULT_MODEL = \"deepseek-v4-flash\"\nDEEPSEEK_API_URL = \"https://api.deepseek.com/v1/chat/completions\"\n\n# Back-compat module-level defaults (used only if no provider resolution happens)\nDEFAULT_MODEL = OPENAI_DEFAULT_MODEL\nDEFAULT_BATCH_SIZE = 24\nAPI_URL = OPENAI_API_URL\n\n\ndef zh_ratio(text: str) -> float:\n if not text:\n return 0.0\n zh = sum(1 for ch in text if \"\\u4e00\" <= ch <= \"\\u9fff\")\n return zh / len(text)\n\n\ndef needs_translation(text: str, *, for_description: bool) -> bool:\n text = (text or \"\").strip()\n if not text:\n return False\n ratio = zh_ratio(text)\n if for_description:\n return len(text) > 8 and ratio < 0.18\n return ratio < 0.08\n\n\ndef load_json(path: Path) -> dict[str, Any]:\n return json.loads(path.read_text(encoding=\"utf-8\"))\n\n\ndef save_json(path: Path, obj: object) -> None:\n path.write_text(json.dumps(obj, ensure_ascii=False, indent=2), encoding=\"utf-8\")\n\n\ndef collect_strings(data: dict[str, Any], fields: str) -> list[dict[str, str]]:\n collected: list[dict[str, str]] = []\n seen: set[tuple[str, str]] = set()\n include_n = fields in {\"n\", \"both\"}\n include_d = fields in {\"d\", \"both\"}\n\n for value in data.values():\n if not isinstance(value, dict):\n continue\n if include_n:\n name = value.get(\"n\")\n if isinstance(name, str) and needs_translation(name, for_description=False):\n key = (\"n\", name.strip())\n if key not in seen:\n seen.add(key)\n collected.append({\"field\": \"n\", \"source\": name.strip()})\n if include_d:\n desc = value.get(\"d\")\n if isinstance(desc, str) and needs_translation(desc, for_description=True):\n key = (\"d\", desc.strip())\n if key not in seen:\n seen.add(key)\n collected.append({\"field\": \"d\", \"source\": desc.strip()})\n return collected\n\n\ndef apply_cache(data: dict[str, Any], cache: dict[str, str]) -> None:\n for value in data.values():\n if not isinstance(value, dict):\n continue\n for field in (\"n\", \"d\"):\n src = value.get(field)\n if isinstance(src, str):\n key = f\"{field}\\t{src.strip()}\"\n if key in cache:\n value[field] = cache[key]\n\n\ndef build_schema() -> dict[str, Any]:\n return {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"items\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"id\": {\"type\": \"string\"},\n \"translation\": {\"type\": \"string\"},\n },\n \"required\": [\"id\", \"translation\"],\n },\n }\n },\n \"required\": [\"items\"],\n }\n\n\ndef load_glossary() -> dict[str, Any]:\n if GLOSSARY_PATH.is_file():\n try:\n return json.loads(GLOSSARY_PATH.read_text(encoding=\"utf-8\"))\n except Exception:\n return {}\n return {}\n\n\ndef build_strong_d_only_prompt(glossary: dict[str, Any]) -> str:\n \"\"\"构建仅翻译 d 字段的强 Prompt，注入本 GCS 术语表。\"\"\"\n terms = glossary.get(\"terms\", {})\n phrases = glossary.get(\"phrases\", {})\n notes = glossary.get(\"notes_for_ai\", [])\n\n term_lines = \"\\n\".join(f\"- {k} → {v}\" for k, v in terms.items())\n phrase_lines = \"\\n\".join(f\"- \\\"{k}\\\" → \\\"{v}\\\"\" for k, v in phrases.items())\n note_lines = \"\\n\".join(f\"- {n}\" for n in notes)\n\n return (\n \"You are a professional technical translator for ArduPilot parameter documentation.\\n\"\n \"You are translating ONLY the long description field ('d') into clear, accurate Simplified Chinese \"\n \"for a professional Ground Control Station UI used by pilots and engineers.\\n\\n\"\n \"MANDATORY TERMINOLOGY (必须严格遵守):\\n\"\n f\"{term_lines}\\n\\n\"\n \"PREFERRED PHRASES:\\n\"\n f\"{phrase_lines}\\n\\n\"\n \"CRITICAL RULES:\\n\"\n \"- Translate ONLY the 'd' (description) content. Never touch parameter names, mode names (ACRO, FBWA, CIRCLE, RTL, etc.), abbreviations (PID, EKF, INS, GPS, PWM, MAVLink), numbers, units, or ranges.\\n\"\n \"- Preserve all paragraph breaks, list structures, indentation, and formatting intent exactly.\\n\"\n \"- For bitmask definitions, use the exact format: '位0: ...', '位1: ...'\\n\"\n \"- For values lists, keep the original structure and translate labels naturally.\\n\"\n \"- Tone: professional, concise, suitable for experienced pilots and engineers. No added explanations, warnings, or Markdown.\\n\"\n \"- Do not translate the short display name field ('n').\\n\\n\"\n \"QUALITY NOTES:\\n\"\n f\"{note_lines}\\n\"\n )\n\n\ndef write_review_file(review_items: list[dict[str, Any]], review_dir: Path) -> Path:\n \"\"\"Write a timestamped review JSON for human inspection and editing (仅 d 字段)。\"\"\"\n review_dir.mkdir(parents=True, exist_ok=True)\n ts = time.strftime(\"%Y%m%d-%H%M%S\")\n out_path = review_dir / f\"translations.review.{ts}.json\"\n payload = {\n \"_meta\": {\n \"description\": \"Reviewable AI translations — 仅 d（说明）字段。编辑 'proposed' 为最终接受的译文，删除不接受的条目。\",\n \"instructions\": \"完成后运行 apply-reviewed-translations.py 合并到 cache + db（保留 grp）。\",\n \"generated_at\": ts,\n \"count\": len(review_items),\n },\n \"items\": review_items,\n }\n out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding=\"utf-8\")\n return out_path\n\n\n# ---------------------------------------------------------------------------\n# Provider abstraction (OpenAI Responses vs xAI Chat Completions)\n# ---------------------------------------------------------------------------\n\n\ndef resolve_provider(\n *,\n provider_arg: str,\n api_key: str,\n) -> tuple[str, str, str]:\n \"\"\"\n Return (provider, api_url, effective_model_default).\n\n Rules:\n - If provider_arg is \"openai\", \"xai\", or \"deepseek\", honor it explicitly.\n - If \"auto\" (default): auto-detect from key prefix (\"xai-\" → xAI).\n DeepSeek must be selected explicitly via --provider deepseek\n (its keys also start with \"sk-\", same as OpenAI).\n - Returns the appropriate API URL and the recommended default model\n for that provider (caller still respects explicit --model).\n \"\"\"\n p = (provider_arg or \"auto\").lower().strip()\n if p == \"openai\":\n return \"openai\", OPENAI_API_URL, OPENAI_DEFAULT_MODEL\n if p == \"xai\":\n return \"xai\", XAI_API_URL, XAI_DEFAULT_MODEL\n if p == \"deepseek\":\n return \"deepseek\", DEEPSEEK_API_URL, DEEPSEEK_DEFAULT_MODEL\n # auto\n if isinstance(api_key, str) and api_key.strip().lower().startswith(\"xai-\"):\n return \"xai\", XAI_API_URL, XAI_DEFAULT_MODEL\n return \"openai\", OPENAI_API_URL, OPENAI_DEFAULT_MODEL\n\n\ndef build_request_body_for_provider(\n provider: str,\n model: str,\n system_prompt: str,\n user_json_str: str,\n schema: dict[str, Any],\n) -> dict[str, Any]:\n \"\"\"\n Build the JSON body for the chosen provider.\n\n - OpenAI: uses the Responses API.\n - xAI: uses Chat Completions + full json_schema (strict structured outputs).\n - DeepSeek: currently uses Chat Completions + simple \"json_object\" mode\n because full \"json_schema\" + \"strict\" is not yet available on all their\n models/endpoints (including the user's \"deepseek-v4-flash\").\n We rely on a strong prompt + post-parsing instead of server-side enforcement.\n \"\"\"\n if provider == \"deepseek\":\n # DeepSeek: use the widely-supported simple JSON mode.\n # The prompt (especially in --review / d-only mode) already tells the model\n # to return exactly one JSON object with the expected shape.\n return {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_json_str},\n ],\n \"response_format\": {\"type\": \"json_object\"},\n }\n\n if provider == \"xai\":\n # xAI supports the full json_schema structured output.\n return {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_json_str},\n ],\n \"response_format\": {\n \"type\": \"json_schema\",\n \"json_schema\": {\n \"name\": \"param_translation_batch\",\n \"schema\": schema,\n \"strict\": True,\n },\n },\n }\n\n # OpenAI Responses API (current implementation)\n return {\n \"model\": model,\n \"input\": [\n {\n \"role\": \"system\",\n \"content\": [{\"type\": \"input_text\", \"text\": system_prompt}],\n },\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": user_json_str,\n }\n ],\n },\n ],\n \"text\": {\n \"format\": {\n \"type\": \"json_schema\",\n \"name\": \"param_translation_batch\",\n \"schema\": schema,\n \"strict\": True,\n }\n },\n }\n\n\ndef extract_text_from_provider_response(\n provider: str, response_json: dict[str, Any]\n) -> str:\n \"\"\"\n Extract the model's text output from either provider's response shape.\n For structured outputs the text is a JSON string that the caller will parse.\n \"\"\"\n if provider in (\"xai\", \"deepseek\"):\n # Standard Chat Completions shape (used by both xAI legacy path and DeepSeek)\n choices = response_json.get(\"choices\")\n if isinstance(choices, list) and choices:\n msg = choices[0].get(\"message\") if isinstance(choices[0], dict) else None\n if isinstance(msg, dict):\n content = msg.get(\"content\")\n if isinstance(content, str) and content.strip():\n return content.strip()\n if \"error\" in response_json:\n prov = \"DeepSeek\" if provider == \"deepseek\" else \"xAI\"\n raise RuntimeError(f\"{prov} API error: {response_json['error']}\")\n raise RuntimeError(f\"{provider.capitalize()} Chat Completions returned no readable message content\")\n\n # OpenAI Responses (existing logic)\n output_text = response_json.get(\"output_text\")\n if isinstance(output_text, str) and output_text.strip():\n return output_text\n\n output = response_json.get(\"output\")\n if isinstance(output, list):\n chunks: list[str] = []\n for item in output:\n if not isinstance(item, dict):\n continue\n content = item.get(\"content\")\n if not isinstance(content, list):\n continue\n for part in content:\n if not isinstance(part, dict):\n continue\n text = part.get(\"text\")\n if isinstance(text, str):\n chunks.append(text)\n if chunks:\n return \"\\n\".join(chunks)\n\n raise RuntimeError(\"OpenAI Responses API returned no readable text output\")\n\n\n# ---------------------------------------------------------------------------\n# Robust JSON extraction helpers (especially important for DeepSeek)\n# ---------------------------------------------------------------------------\n\n\ndef _strip_code_fences(text: str) -> str:\n \"\"\"Remove common ```json ... ``` or ``` ... ``` fences that models often emit anyway.\"\"\"\n t = text.strip()\n if t.startswith(\"```\"):\n # drop the opening fence line\n lines = t.splitlines()\n if len(lines) > 1:\n t = \"\\n\".join(lines[1:])\n # drop the closing fence if present\n if t.rstrip().endswith(\"```\"):\n t = t.rstrip()[:-3].rstrip()\n return t.strip()\n\n\ndef _extract_outermost_json_object(text: str) -> str | None:\n \"\"\"\n Very simple but effective heuristic: take the substring from the first '{'\n to the last '}'. This recovers the main JSON object even if the model\n added prefix/suffix text or multiple objects.\n \"\"\"\n first = text.find(\"{\")\n last = text.rfind(\"}\")\n if first == -1 or last == -1 or last <= first:\n return None\n return text[first : last + 1]\n\n\ndef normalize_translation_json(provider: str, raw_text: str) -> dict[str, Any]:\n \"\"\"\n Convert whatever the model emitted into a dict that (ideally) has an \"items\" list\n in the shape we expect: {\"items\": [{\"id\": \"...\", \"translation\": \"...\"}, ...]}\n\n For providers using strict json_schema (currently xAI in our flow) we are strict.\n For DeepSeek (json_object mode) we are very forgiving and try several recovery strategies.\n \"\"\"\n if provider != \"deepseek\":\n # Strict path – the schema enforcement should have done its job.\n return json.loads(raw_text)\n\n # --- DeepSeek loose path ---\n cleaned = _strip_code_fences(raw_text)\n\n # Try direct parse first\n try:\n data = json.loads(cleaned)\n except json.JSONDecodeError:\n # Salvage attempt: grab the biggest {...} block\n candidate = _extract_outermost_json_object(cleaned)\n if candidate:\n data = json.loads(candidate)\n else:\n # Give up – surface the raw text for debugging\n raise ValueError(\n f\"DeepSeek did not return valid JSON. Raw model output (first 1200 chars):\\n{cleaned[:1200]}\"\n )\n\n # Normalize several shapes that DeepSeek commonly produces\n if isinstance(data, list):\n # Model returned the array directly instead of an object\n data = {\"items\": data}\n\n if not isinstance(data, dict):\n data = {}\n\n if \"items\" not in data or not isinstance(data.get(\"items\"), list):\n # Try common alternative shapes\n if \"translations\" in data and isinstance(data[\"translations\"], dict):\n # {\"translations\": {\"d-1\": \"foo\", \"d-2\": \"bar\", ...}}\n items = [\n {\"id\": k, \"translation\": v}\n for k, v in data[\"translations\"].items()\n if isinstance(k, str) and isinstance(v, str)\n ]\n data = {\"items\": items}\n\n elif all(isinstance(k, str) and isinstance(v, str) for k, v in data.items()):\n # Flat dict: {\"d-1\": \"foo\", \"d-2\": \"bar\"}\n items = [{\"id\": k, \"translation\": v} for k, v in data.items()]\n data = {\"items\": items}\n\n # If we still don't have a usable \"items\", keep whatever we have so the\n # caller can produce a good error message that includes the raw output.\n\n return data\n\n\ndef extract_output_text(response_json: dict[str, Any]) -> str:\n \"\"\"Legacy wrapper kept for any direct callers; delegates to OpenAI path.\"\"\"\n return extract_text_from_provider_response(\"openai\", response_json)\n\n\ndef translate_batch(\n *,\n api_key: str,\n model: str,\n batch: list[dict[str, str]],\n timeout_s: float,\n glossary: dict[str, Any] | None = None,\n force_d_only: bool = False,\n provider: str = \"openai\",\n) -> dict[str, str]:\n \"\"\"\n Call the chosen provider (openai, xai, or deepseek) to translate a batch.\n\n The provider determines:\n - Which API endpoint and request shape is used (via build_request_body_for_provider)\n - How the response is parsed (via extract_text_from_provider_response)\n \"\"\"\n glossary = glossary or {}\n if force_d_only or all(item.get(\"field\") == \"d\" for item in batch):\n prompt = build_strong_d_only_prompt(glossary)\n else:\n prompt = (\n \"You are translating ArduPilot parameter metadata from English into Simplified Chinese \"\n \"for a flight-control ground station UI.\\n\"\n \"Rules:\\n\"\n \"1. Keep technical tokens unchanged: parameter names like YAW_RATE_ENABLE, MAVLink, GPS, RC, PWM, ACRO.\\n\"\n \"2. Use accurate avionics/flight-control terminology, not word-for-word machine translation.\\n\"\n \"3. Preserve paragraph breaks, list structure, numeric ranges, units, and punctuation intent.\\n\"\n \"4. The 'n' field should be short, natural UI text.\\n\"\n \"5. The 'd' field should be readable Chinese technical prose for pilots/engineers.\\n\"\n \"6. Do not add explanations, notes, or markdown.\\n\"\n \"7. Return one translation per input item.\\n\"\n )\n\n # When using DeepSeek with the simple \"json_object\" mode (no strict schema enforcement),\n # we must be extremely explicit. We also give a tiny concrete example of the exact shape.\n if provider == \"deepseek\":\n extra = (\n \"\\n\\nCRITICAL OUTPUT FORMAT (MANDATORY):\\n\"\n \"Your entire response MUST be exactly one JSON object and NOTHING ELSE.\\n\"\n \"Do not output any text before or after the JSON.\\n\"\n \"Do not use markdown code fences (```json ... ```).\\n\"\n \"The object must have exactly this shape:\\n\"\n ' {\"items\": [ {\"id\": \"d-1\", \"translation\": \"中文译文...\"}, {\"id\": \"d-2\", \"translation\": \"...\"} ] }\\n'\n \"Use the exact ids we gave you (d-1, d-2, ...). \"\n \"Only translate the 'source' text into clear Simplified Chinese. \"\n \"Output ONLY the raw JSON object described above.\"\n )\n prompt = prompt + extra\n\n input_payload = [\n {\n \"id\": f\"{item['field']}-{idx}\",\n \"field\": item[\"field\"],\n \"source\": item[\"source\"],\n }\n for idx, item in enumerate(batch, start=1)\n ]\n\n user_json_str = json.dumps({\"items\": input_payload}, ensure_ascii=False)\n schema = build_schema()\n\n body = build_request_body_for_provider(\n provider=provider,\n model=model,\n system_prompt=prompt,\n user_json_str=user_json_str,\n schema=schema,\n )\n\n if provider == \"deepseek\":\n api_url = DEEPSEEK_API_URL\n elif provider == \"xai\":\n api_url = XAI_API_URL\n else:\n api_url = OPENAI_API_URL\n\n request = urllib.request.Request(\n api_url,\n data=json.dumps(body).encode(\"utf-8\"),\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n method=\"POST\",\n )\n\n try:\n with urllib.request.urlopen(request, timeout=timeout_s) as response:\n response_json = json.loads(response.read().decode(\"utf-8\"))\n except urllib.error.HTTPError as exc:\n details = exc.read().decode(\"utf-8\", errors=\"replace\")\n if provider == \"deepseek\":\n prov_name = \"DeepSeek\"\n elif provider == \"xai\":\n prov_name = \"xAI\"\n else:\n prov_name = \"OpenAI\"\n raise RuntimeError(f\"{prov_name} API HTTP {exc.code}: {details}\") from exc\n\n raw_text = extract_text_from_provider_response(provider, response_json)\n\n try:\n parsed = normalize_translation_json(provider, raw_text)\n except Exception as exc:\n # For DeepSeek we want to show the user exactly what the model emitted\n if provider == \"deepseek\":\n preview = raw_text[:1500] if raw_text else \"(empty)\"\n raise RuntimeError(\n f\"DeepSeek returned something that could not be parsed as the expected JSON.\\n\"\n f\"Raw model output (first 1500 chars):\\n{preview}\\n\\n\"\n f\"Original error: {exc}\"\n ) from exc\n raise\n\n items = parsed.get(\"items\")\n if not isinstance(items, list):\n hint = \"\"\n if provider == \"deepseek\":\n preview = raw_text[:1200] if raw_text else \"(empty)\"\n hint = (\n f\"\\n\\nDeepSeek raw output (first 1200 chars) for debugging:\\n{preview}\\n\"\n \"The model did not follow the requested JSON shape. \"\n \"Common causes: it added explanations, used a different key name, \"\n \"or wrapped the result in extra text despite the prompt.\"\n )\n elif provider == \"xai\":\n hint = \" (xAI: try a current model such as grok-4.3 or grok-4.20-0309-reasoning that supports structured outputs)\"\n raise RuntimeError(f\"Structured output missing items array{hint}\")\n\n # Collect translations keyed by the id the model returned, while also\n # preserving order so we can fall back to positional matching when the\n # model echoes the wrong ids (a common DeepSeek json_object quirk).\n translations: dict[str, str] = {}\n ordered_translations: list[str] = []\n for item in items:\n if not isinstance(item, dict):\n continue\n item_id = item.get(\"id\")\n translation = item.get(\"translation\")\n if not isinstance(translation, str):\n continue\n translation = translation.strip()\n ordered_translations.append(translation)\n if isinstance(item_id, str):\n translations[item_id] = translation\n\n # Positional fallback: the caller looks up expected ids \"<field>-<idx>\"\n # (1-based, matching input_payload). If the model returned the right number\n # of items in order but with mismatched/renamed ids, recover by position.\n expected_ids = [f\"{item['field']}-{idx}\" for idx, item in enumerate(batch, start=1)]\n for pos, expected_id in enumerate(expected_ids):\n if translations.get(expected_id):\n continue\n if pos < len(ordered_translations) and ordered_translations[pos]:\n translations[expected_id] = ordered_translations[pos]\n\n return translations\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(\n description=\"Translate apm-param-db 'd' (description) field using OpenAI, xAI or DeepSeek. \"\n \"Use --review for safe, reviewable workflow (recommended).\"\n )\n parser.add_argument(\n \"--provider\",\n choices=[\"auto\", \"openai\", \"xai\", \"deepseek\"],\n default=\"auto\",\n help=\"AI provider. 'auto' (default) detects from key prefix (xai- → xAI). \"\n \"Use 'deepseek' for DeepSeek (requires explicit --provider deepseek; \"\n \"recommended with --api-key-env DEEPSEEK_API_KEY).\",\n )\n parser.add_argument(\n \"--model\",\n default=None,\n help=\"Model ID for the chosen provider (default: grok-4.3 for xAI, deepseek-v4-flash for DeepSeek, \"\n \"gpt-5.4-mini for OpenAI). For DeepSeek pass --provider deepseek and optionally --model.\",\n )\n parser.add_argument(\"--limit\", type=int, default=0, help=\"Maximum number of uncached strings to translate\")\n parser.add_argument(\"--batch-size\", type=int, default=DEFAULT_BATCH_SIZE, help=\"Strings per API call\")\n parser.add_argument(\"--sleep\", type=float, default=0.2, help=\"Sleep between API calls in seconds\")\n parser.add_argument(\n \"--max-retries\",\n type=int,\n default=3,\n help=\"Attempts per batch before giving up (handles flaky/incomplete API responses). Default: 3\",\n )\n parser.add_argument(\n \"--skip-failed\",\n action=\"store_true\",\n help=\"Skip a batch that is still incomplete after retries instead of aborting the run \"\n \"(missing items remain uncached and are retried on the next run).\",\n )\n parser.add_argument(\"--fields\", choices=[\"n\", \"d\", \"both\"], default=\"both\", help=\"Which fields to translate\")\n parser.add_argument(\n \"--review\",\n action=\"store_true\",\n help=\"Reviewable mode: generate editable review JSON files for 'd' only (strongly recommended for quality)\",\n )\n parser.add_argument(\"--review-dir\", default=str(REVIEW_DIR), help=\"Directory for review JSON files\")\n parser.add_argument(\"--timeout\", type=float, default=90.0, help=\"Per-request timeout in seconds\")\n parser.add_argument(\n \"--api-key-env\",\n default=\"OPENAI_API_KEY\",\n help=\"Environment variable holding the API key. For xAI prefer XAI_API_KEY and --api-key-env XAI_API_KEY\",\n )\n parser.add_argument(\"--cache-path\", default=str(AI_CACHE_PATH), help=\"Translation cache JSON path\")\n parser.add_argument(\"--source-path\", default=str(SOURCE_PATH), help=\"English source DB path\")\n parser.add_argument(\"--output-path\", default=str(DB_PATH), help=\"Translated output DB path\")\n parser.add_argument(\"--skip-backup\", action=\"store_true\", help=\"Do not write a pre-AI backup of the output file\")\n args = parser.parse_args()\n\n api_key = os.environ.get(args.api_key_env, \"\").strip()\n if not api_key:\n raise SystemExit(\n f\"Missing API key: set {args.api_key_env}\\n\"\n \" For DeepSeek: export DEEPSEEK_API_KEY=... then use --api-key-env DEEPSEEK_API_KEY --provider deepseek\\n\"\n \" For xAI: export XAI_API_KEY=... then use --api-key-env XAI_API_KEY\\n\"\n \" For OpenAI: export OPENAI_API_KEY=...\"\n )\n\n # Hardening against the very common \"copy-paste pollution\" problem on non-English systems.\n # API keys must be pure ASCII. Non-ASCII in the Authorization header causes the opaque\n # \"latin-1 codec can't encode\" crash deep inside urllib/http.client.\n if not api_key.isascii():\n # Show a short repr of the offending characters to help the user locate the garbage.\n bad = [(i, c, hex(ord(c))) for i, c in enumerate(api_key) if not c.isascii()]\n preview = api_key[:8] + \"...\" + api_key[-4:] if len(api_key) > 12 else api_key\n raise SystemExit(\n \"API key contains non-ASCII characters (very common after copy-paste from chat/browser).\\n\"\n f\" Variable : {args.api_key_env}\\n\"\n f\" Preview : {preview}\\n\"\n f\" Bad chars: {bad[:5]}{'...' if len(bad) > 5 else ''}\\n\\n\"\n \"Please re-export a clean key:\\n\"\n f\" export {args.api_key_env}='xai-XXXXXXXXXXXXXXXX...'\\n\"\n \" (use straight single quotes, paste ONLY the key, no extra characters, no Chinese punctuation)\\n\"\n \"Then re-run the command.\"\n )\n\n # Resolve provider (auto-detect from key prefix if --provider=auto)\n provider, _api_url, provider_default_model = resolve_provider(\n provider_arg=getattr(args, \"provider\", \"auto\"),\n api_key=api_key,\n )\n\n # Effective model: explicit --model wins, otherwise the provider's recommended default\n effective_model = args.model or provider_default_model\n\n source_path = Path(args.source_path)\n output_path = Path(args.output_path)\n cache_path = Path(args.cache_path)\n\n if not source_path.is_file():\n if source_path == SOURCE_PATH and DB_PATH.is_file():\n raise SystemExit(\n f\"Missing English source DB: {source_path}\\n\"\n f\"Current translated DB exists at {DB_PATH}. Please restore or provide --source-path.\"\n )\n raise SystemExit(f\"Missing source DB: {source_path}\")\n\n source_data = load_json(source_path)\n if not isinstance(source_data, dict):\n raise SystemExit(\"Source DB root is not an object\")\n\n cache: dict[str, str] = {}\n if cache_path.is_file():\n raw_cache = load_json(cache_path)\n if isinstance(raw_cache, dict):\n cache = {str(k): str(v) for k, v in raw_cache.items()}\n\n glossary = load_glossary()\n\n # In review mode we focus exclusively on 'd' (说明) as per project decision\n effective_fields = \"d\" if args.review else args.fields\n\n pending_all = []\n for item in collect_strings(source_data, effective_fields):\n cache_key = f\"{item['field']}\\t{item['source']}\"\n if cache_key not in cache:\n pending_all.append(item)\n\n pending = pending_all[: args.limit] if args.limit > 0 else pending_all\n\n print(\n f\"Provider: {provider} | model: {effective_model} | \"\n f\"Source entries: {len(source_data)} | uncached pending: {len(pending_all)}\"\n + (f\" | this run: {len(pending)}\" if args.limit > 0 else \"\")\n + (f\" | REVIEW MODE (d only)\" if args.review else \"\"),\n flush=True,\n )\n\n if not pending:\n print(\"Nothing to translate.\", flush=True)\n return\n\n done = 0\n review_items: list[dict[str, Any]] = []\n\n for start in range(0, len(pending), max(1, args.batch_size)):\n batch = pending[start : start + max(1, args.batch_size)]\n\n # Retry a flaky batch a few times before giving up. A single bad\n # response (e.g. DeepSeek returning mismatched ids or dropping items)\n # should not abort the whole run and lose hundreds of cached results.\n translations: dict[str, str] = {}\n missing: list[str] = []\n last_error: Exception | None = None\n for attempt in range(1, max(1, args.max_retries) + 1):\n try:\n translations = translate_batch(\n api_key=api_key,\n model=effective_model,\n batch=batch,\n timeout_s=args.timeout,\n glossary=glossary,\n force_d_only=args.review,\n provider=provider,\n )\n except Exception as exc: # transient API/parse failures\n last_error = exc\n translations = {}\n else:\n last_error = None\n\n missing = [\n f\"{item['field']}-{idx}\"\n for idx, item in enumerate(batch, start=1)\n if not translations.get(f\"{item['field']}-{idx}\", \"\").strip()\n ]\n if not missing and last_error is None:\n break\n\n if attempt < max(1, args.max_retries):\n backoff = args.sleep * (2 ** (attempt - 1)) + 0.5\n reason = (\n f\"error: {last_error}\" if last_error is not None\n else f\"incomplete ({len(missing)} missing)\"\n )\n print(\n f\" Batch at {start} {reason}; retry {attempt}/{args.max_retries - 1} \"\n f\"in {backoff:.1f}s\",\n flush=True,\n )\n time.sleep(backoff)\n\n if last_error is not None:\n raise RuntimeError(\n f\"Batch at offset {start} failed after {args.max_retries} attempts: {last_error}\"\n ) from last_error\n\n for idx, item in enumerate(batch, start=1):\n response_id = f\"{item['field']}-{idx}\"\n translated = translations.get(response_id, \"\").strip()\n if not translated:\n continue\n\n cache_key = f\"{item['field']}\\t{item['source']}\"\n cache[cache_key] = translated\n done += 1\n\n if args.review and item[\"field\"] == \"d\":\n # Try to enrich with param name (source_data keys are the param names)\n param_name = None\n for pname, pval in source_data.items():\n if isinstance(pval, dict) and pval.get(\"d\") == item[\"source\"]:\n param_name = pname\n break\n review_items.append({\n \"id\": response_id,\n \"param\": param_name or \"UNKNOWN\",\n \"field\": \"d\",\n \"source\": item[\"source\"],\n \"proposed\": translated,\n \"status\": \"pending\", # user should change to \"accepted\" or delete\n })\n\n if missing:\n msg = (\n f\"Batch at offset {start} still incomplete after {args.max_retries} attempts \"\n f\"({len(missing)} missing): {', '.join(missing)}\"\n )\n if args.skip_failed:\n print(f\" WARNING: {msg} — skipping (will be retried on next run).\", flush=True)\n else:\n raise RuntimeError(msg + \"\\nUse --skip-failed to continue past bad batches.\")\n\n save_json(cache_path, cache)\n print(f\"Translated {done}/{len(pending)}\", flush=True)\n time.sleep(max(0.0, args.sleep))\n\n review_dir = Path(args.review_dir)\n if args.review and review_items:\n review_path = write_review_file(review_items, review_dir)\n print(f\"\\nReview file written: {review_path}\", flush=True)\n print(\"Please review the 'proposed' translations, edit as needed, then run the apply tool.\", flush=True)\n print(\"No changes were written to the final db in review mode.\", flush=True)\n return\n\n # Normal (non-review) path — apply to output\n output_data = load_json(source_path)\n apply_cache(output_data, cache)\n\n if output_path.is_file() and not args.skip_backup:\n shutil.copy2(output_path, BACKUP_OUT_PATH)\n\n save_json(output_path, output_data)\n print(f\"Done. Updated {output_path} with {done} new AI translations.\", flush=True)\n\n\nif __name__ == \"__main__\":\n try:\n main()\n except KeyboardInterrupt:\n sys.exit(130)\n"}, "78832cb2a41a7127c1977fcaf3acfbee1e626f19": {"t": 1785763579.8874643, "n": 35813, "h": "ccb6c0a548af1f61", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nTranslate ArduPilot parameter metadata using OpenAI (Responses API) or xAI (Chat Completions).\n\nThis script reads English source strings from JS/data/apm-param-db.en.json,\ntranslates them (primarily the long 'd' description field) into Simplified Chinese\nin batches, caches results locally, and writes the merged output back to\nJS/data/apm-param-db.json.\n\nSupported providers:\n - OpenAI (default for keys not starting with \"xai-\")\n - xAI (Grok) — auto-detected when the API key starts with \"xai-\",\n or explicitly selected with --provider xai. Recommended default: grok-4.3\n - DeepSeek — explicitly selected with --provider deepseek.\n Recommended default: deepseek-v4-flash (user-confirmed working model)\n\nEnvironment variables (recommended):\n OPENAI_API_KEY For OpenAI\n XAI_API_KEY For xAI (Grok)\n DEEPSEEK_API_KEY For DeepSeek\n\nExample (xAI):\n export XAI_API_KEY=\"xai-...\"\n python translate_params_ai.py --review --limit 300 --api-key-env XAI_API_KEY\n\nExample (DeepSeek):\n export DEEPSEEK_API_KEY=\"sk-...\"\n python translate_params_ai.py --review --limit 300 \\\n --api-key-env DEEPSEEK_API_KEY --provider deepseek --model deepseek-v4-flash\n\nExample (OpenAI):\n export OPENAI_API_KEY=\"sk-...\"\n python translate_params_ai.py --model gpt-5.4-mini --limit 200\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport os\nimport shutil\nimport sys\nimport time\nimport urllib.error\nimport urllib.request\nfrom pathlib import Path\nfrom typing import Any\n\nROOT = Path(__file__).resolve().parent\nDB_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.json\"\nSOURCE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.en.json\"\nLEGACY_CACHE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-translate-cache.json\"\nAI_CACHE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-translate-cache.ai.json\"\nBACKUP_OUT_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.pre-ai-backup.json\"\n\n# Reviewable workflow artifacts (仅 d 字段)\nREVIEW_DIR = ROOT / \"tools\" / \"translation-review\"\nGLOSSARY_PATH = ROOT / \"tools\" / \"glossary-param-translation.json\"\n\n# Provider-aware defaults\nOPENAI_DEFAULT_MODEL = \"gpt-5.4-mini\"\nOPENAI_API_URL = \"https://api.openai.com/v1/responses\"\n\n# Current recommended default for xAI (as of late May 2026).\n# grok-2 / grok-3 and early grok-4 variants were retired on 2026-05-15.\n# grok-4.3 is the current flagship (best quality/speed for technical translation).\n# For heavier reasoning on complex descriptions you may override with:\n# --model grok-4.20-0309-reasoning\nXAI_DEFAULT_MODEL = \"grok-4.3\"\nXAI_API_URL = \"https://api.x.ai/v1/chat/completions\"\n\n# DeepSeek (OpenAI-compatible Chat Completions + json_schema support)\n# User confirmed \"deepseek-v4-flash\" works for their use case.\n# Standard base for DeepSeek is https://api.deepseek.com\nDEEPSEEK_DEFAULT_MODEL = \"deepseek-v4-flash\"\nDEEPSEEK_API_URL = \"https://api.deepseek.com/v1/chat/completions\"\n\n# Back-compat module-level defaults (used only if no provider resolution happens)\nDEFAULT_MODEL = OPENAI_DEFAULT_MODEL\nDEFAULT_BATCH_SIZE = 24\nAPI_URL = OPENAI_API_URL\n\n\ndef zh_ratio(text: str) -> float:\n if not text:\n return 0.0\n zh = sum(1 for ch in text if \"\\u4e00\" <= ch <= \"\\u9fff\")\n return zh / len(text)\n\n\ndef needs_translation(text: str, *, for_description: bool) -> bool:\n text = (text or \"\").strip()\n if not text:\n return False\n ratio = zh_ratio(text)\n if for_description:\n return len(text) > 8 and ratio < 0.18\n return ratio < 0.08\n\n\ndef load_json(path: Path) -> dict[str, Any]:\n return json.loads(path.read_text(encoding=\"utf-8\"))\n\n\ndef save_json(path: Path, obj: object) -> None:\n path.write_text(json.dumps(obj, ensure_ascii=False, indent=2), encoding=\"utf-8\")\n\n\ndef collect_strings(data: dict[str, Any], fields: str) -> list[dict[str, str]]:\n collected: list[dict[str, str]] = []\n seen: set[tuple[str, str]] = set()\n include_n = fields in {\"n\", \"both\"}\n include_d = fields in {\"d\", \"both\"}\n\n for value in data.values():\n if not isinstance(value, dict):\n continue\n if include_n:\n name = value.get(\"n\")\n if isinstance(name, str) and needs_translation(name, for_description=False):\n key = (\"n\", name.strip())\n if key not in seen:\n seen.add(key)\n collected.append({\"field\": \"n\", \"source\": name.strip()})\n if include_d:\n desc = value.get(\"d\")\n if isinstance(desc, str) and needs_translation(desc, for_description=True):\n key = (\"d\", desc.strip())\n if key not in seen:\n seen.add(key)\n collected.append({\"field\": \"d\", \"source\": desc.strip()})\n return collected\n\n\ndef apply_cache(data: dict[str, Any], cache: dict[str, str]) -> None:\n for value in data.values():\n if not isinstance(value, dict):\n continue\n for field in (\"n\", \"d\"):\n src = value.get(field)\n if isinstance(src, str):\n key = f\"{field}\\t{src.strip()}\"\n if key in cache:\n value[field] = cache[key]\n\n\ndef build_schema() -> dict[str, Any]:\n return {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"items\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"id\": {\"type\": \"string\"},\n \"translation\": {\"type\": \"string\"},\n },\n \"required\": [\"id\", \"translation\"],\n },\n }\n },\n \"required\": [\"items\"],\n }\n\n\ndef load_glossary() -> dict[str, Any]:\n if GLOSSARY_PATH.is_file():\n try:\n return json.loads(GLOSSARY_PATH.read_text(encoding=\"utf-8\"))\n except Exception:\n return {}\n return {}\n\n\ndef build_strong_d_only_prompt(glossary: dict[str, Any]) -> str:\n \"\"\"构建仅翻译 d 字段的强 Prompt，注入本 GCS 术语表。\"\"\"\n terms = glossary.get(\"terms\", {})\n phrases = glossary.get(\"phrases\", {})\n notes = glossary.get(\"notes_for_ai\", [])\n\n term_lines = \"\\n\".join(f\"- {k} → {v}\" for k, v in terms.items())\n phrase_lines = \"\\n\".join(f\"- \\\"{k}\\\" → \\\"{v}\\\"\" for k, v in phrases.items())\n note_lines = \"\\n\".join(f\"- {n}\" for n in notes)\n\n return (\n \"You are a professional technical translator for ArduPilot parameter documentation.\\n\"\n \"You are translating ONLY the long description field ('d') into clear, accurate Simplified Chinese \"\n \"for a professional Ground Control Station UI used by pilots and engineers.\\n\\n\"\n \"MANDATORY TERMINOLOGY (必须严格遵守):\\n\"\n f\"{term_lines}\\n\\n\"\n \"PREFERRED PHRASES:\\n\"\n f\"{phrase_lines}\\n\\n\"\n \"CRITICAL RULES:\\n\"\n \"- Translate ONLY the 'd' (description) content. Never touch parameter names, mode names (ACRO, FBWA, CIRCLE, RTL, etc.), abbreviations (PID, EKF, INS, GPS, PWM, MAVLink), numbers, units, or ranges.\\n\"\n \"- Preserve all paragraph breaks, list structures, indentation, and formatting intent exactly.\\n\"\n \"- For bitmask definitions, use the exact format: '位0: ...', '位1: ...'\\n\"\n \"- For values lists, keep the original structure and translate labels naturally.\\n\"\n \"- Tone: professional, concise, suitable for experienced pilots and engineers. No added explanations, warnings, or Markdown.\\n\"\n \"- Do not translate the short display name field ('n').\\n\\n\"\n \"QUALITY NOTES:\\n\"\n f\"{note_lines}\\n\"\n )\n\n\ndef write_review_file(review_items: list[dict[str, Any]], review_dir: Path) -> Path:\n \"\"\"Write a timestamped review JSON for human inspection and editing (仅 d 字段)。\"\"\"\n review_dir.mkdir(parents=True, exist_ok=True)\n ts = time.strftime(\"%Y%m%d-%H%M%S\")\n out_path = review_dir / f\"translations.review.{ts}.json\"\n payload = {\n \"_meta\": {\n \"description\": \"Reviewable AI translations — 仅 d（说明）字段。编辑 'proposed' 为最终接受的译文，删除不接受的条目。\",\n \"instructions\": \"完成后运行 apply-reviewed-translations.py 合并到 cache + db（保留 grp）。\",\n \"generated_at\": ts,\n \"count\": len(review_items),\n },\n \"items\": review_items,\n }\n out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding=\"utf-8\")\n return out_path\n\n\n# ---------------------------------------------------------------------------\n# Provider abstraction (OpenAI Responses vs xAI Chat Completions)\n# ---------------------------------------------------------------------------\n\n\ndef resolve_provider(\n *,\n provider_arg: str,\n api_key: str,\n) -> tuple[str, str, str]:\n \"\"\"\n Return (provider, api_url, effective_model_default).\n\n Rules:\n - If provider_arg is \"openai\", \"xai\", or \"deepseek\", honor it explicitly.\n - If \"auto\" (default): auto-detect from key prefix (\"xai-\" → xAI).\n DeepSeek must be selected explicitly via --provider deepseek\n (its keys also start with \"sk-\", same as OpenAI).\n - Returns the appropriate API URL and the recommended default model\n for that provider (caller still respects explicit --model).\n \"\"\"\n p = (provider_arg or \"auto\").lower().strip()\n if p == \"openai\":\n return \"openai\", OPENAI_API_URL, OPENAI_DEFAULT_MODEL\n if p == \"xai\":\n return \"xai\", XAI_API_URL, XAI_DEFAULT_MODEL\n if p == \"deepseek\":\n return \"deepseek\", DEEPSEEK_API_URL, DEEPSEEK_DEFAULT_MODEL\n # auto\n if isinstance(api_key, str) and api_key.strip().lower().startswith(\"xai-\"):\n return \"xai\", XAI_API_URL, XAI_DEFAULT_MODEL\n return \"openai\", OPENAI_API_URL, OPENAI_DEFAULT_MODEL\n\n\ndef build_request_body_for_provider(\n provider: str,\n model: str,\n system_prompt: str,\n user_json_str: str,\n schema: dict[str, Any],\n) -> dict[str, Any]:\n \"\"\"\n Build the JSON body for the chosen provider.\n\n - OpenAI: uses the Responses API.\n - xAI: uses Chat Completions + full json_schema (strict structured outputs).\n - DeepSeek: currently uses Chat Completions + simple \"json_object\" mode\n because full \"json_schema\" + \"strict\" is not yet available on all their\n models/endpoints (including the user's \"deepseek-v4-flash\").\n We rely on a strong prompt + post-parsing instead of server-side enforcement.\n \"\"\"\n if provider == \"deepseek\":\n # DeepSeek: use the widely-supported simple JSON mode.\n # The prompt (especially in --review / d-only mode) already tells the model\n # to return exactly one JSON object with the expected shape.\n return {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_json_str},\n ],\n \"response_format\": {\"type\": \"json_object\"},\n }\n\n if provider == \"xai\":\n # xAI supports the full json_schema structured output.\n return {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_json_str},\n ],\n \"response_format\": {\n \"type\": \"json_schema\",\n \"json_schema\": {\n \"name\": \"param_translation_batch\",\n \"schema\": schema,\n \"strict\": True,\n },\n },\n }\n\n # OpenAI Responses API (current implementation)\n return {\n \"model\": model,\n \"input\": [\n {\n \"role\": \"system\",\n \"content\": [{\"type\": \"input_text\", \"text\": system_prompt}],\n },\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": user_json_str,\n }\n ],\n },\n ],\n \"text\": {\n \"format\": {\n \"type\": \"json_schema\",\n \"name\": \"param_translation_batch\",\n \"schema\": schema,\n \"strict\": True,\n }\n },\n }\n\n\ndef extract_text_from_provider_response(\n provider: str, response_json: dict[str, Any]\n) -> str:\n \"\"\"\n Extract the model's text output from either provider's response shape.\n For structured outputs the text is a JSON string that the caller will parse.\n \"\"\"\n if provider in (\"xai\", \"deepseek\"):\n # Standard Chat Completions shape (used by both xAI legacy path and DeepSeek)\n choices = response_json.get(\"choices\")\n if isinstance(choices, list) and choices:\n msg = choices[0].get(\"message\") if isinstance(choices[0], dict) else None\n if isinstance(msg, dict):\n content = msg.get(\"content\")\n if isinstance(content, str) and content.strip():\n return content.strip()\n if \"error\" in response_json:\n prov = \"DeepSeek\" if provider == \"deepseek\" else \"xAI\"\n raise RuntimeError(f\"{prov} API error: {response_json['error']}\")\n raise RuntimeError(f\"{provider.capitalize()} Chat Completions returned no readable message content\")\n\n # OpenAI Responses (existing logic)\n output_text = response_json.get(\"output_text\")\n if isinstance(output_text, str) and output_text.strip():\n return output_text\n\n output = response_json.get(\"output\")\n if isinstance(output, list):\n chunks: list[str] = []\n for item in output:\n if not isinstance(item, dict):\n continue\n content = item.get(\"content\")\n if not isinstance(content, list):\n continue\n for part in content:\n if not isinstance(part, dict):\n continue\n text = part.get(\"text\")\n if isinstance(text, str):\n chunks.append(text)\n if chunks:\n return \"\\n\".join(chunks)\n\n raise RuntimeError(\"OpenAI Responses API returned no readable text output\")\n\n\n# ---------------------------------------------------------------------------\n# Robust JSON extraction helpers (especially important for DeepSeek)\n# ---------------------------------------------------------------------------\n\n\ndef _strip_code_fences(text: str) -> str:\n \"\"\"Remove common ```json ... ``` or ``` ... ``` fences that models often emit anyway.\"\"\"\n t = text.strip()\n if t.startswith(\"```\"):\n # drop the opening fence line\n lines = t.splitlines()\n if len(lines) > 1:\n t = \"\\n\".join(lines[1:])\n # drop the closing fence if present\n if t.rstrip().endswith(\"```\"):\n t = t.rstrip()[:-3].rstrip()\n return t.strip()\n\n\ndef _extract_outermost_json_object(text: str) -> str | None:\n \"\"\"\n Very simple but effective heuristic: take the substring from the first '{'\n to the last '}'. This recovers the main JSON object even if the model\n added prefix/suffix text or multiple objects.\n \"\"\"\n first = text.find(\"{\")\n last = text.rfind(\"}\")\n if first == -1 or last == -1 or last <= first:\n return None\n return text[first : last + 1]\n\n\ndef normalize_translation_json(provider: str, raw_text: str) -> dict[str, Any]:\n \"\"\"\n Convert whatever the model emitted into a dict that (ideally) has an \"items\" list\n in the shape we expect: {\"items\": [{\"id\": \"...\", \"translation\": \"...\"}, ...]}\n\n For providers using strict json_schema (currently xAI in our flow) we are strict.\n For DeepSeek (json_object mode) we are very forgiving and try several recovery strategies.\n \"\"\"\n if provider != \"deepseek\":\n # Strict path – the schema enforcement should have done its job.\n return json.loads(raw_text)\n\n # --- DeepSeek loose path ---\n cleaned = _strip_code_fences(raw_text)\n\n # Try direct parse first\n try:\n data = json.loads(cleaned)\n except json.JSONDecodeError:\n # Salvage attempt: grab the biggest {...} block\n candidate = _extract_outermost_json_object(cleaned)\n if candidate:\n data = json.loads(candidate)\n else:\n # Give up – surface the raw text for debugging\n raise ValueError(\n f\"DeepSeek did not return valid JSON. Raw model output (first 1200 chars):\\n{cleaned[:1200]}\"\n )\n\n # Normalize several shapes that DeepSeek commonly produces\n if isinstance(data, list):\n # Model returned the array directly instead of an object\n data = {\"items\": data}\n\n if not isinstance(data, dict):\n data = {}\n\n if \"items\" not in data or not isinstance(data.get(\"items\"), list):\n # Try common alternative shapes\n if \"translations\" in data and isinstance(data[\"translations\"], dict):\n # {\"translations\": {\"d-1\": \"foo\", \"d-2\": \"bar\", ...}}\n items = [\n {\"id\": k, \"translation\": v}\n for k, v in data[\"translations\"].items()\n if isinstance(k, str) and isinstance(v, str)\n ]\n data = {\"items\": items}\n\n elif all(isinstance(k, str) and isinstance(v, str) for k, v in data.items()):\n # Flat dict: {\"d-1\": \"foo\", \"d-2\": \"bar\"}\n items = [{\"id\": k, \"translation\": v} for k, v in data.items()]\n data = {\"items\": items}\n\n # If we still don't have a usable \"items\", keep whatever we have so the\n # caller can produce a good error message that includes the raw output.\n\n return data\n\n\ndef extract_output_text(response_json: dict[str, Any]) -> str:\n \"\"\"Legacy wrapper kept for any direct callers; delegates to OpenAI path.\"\"\"\n return extract_text_from_provider_response(\"openai\", response_json)\n\n\ndef translate_batch(\n *,\n api_key: str,\n model: str,\n batch: list[dict[str, str]],\n timeout_s: float,\n glossary: dict[str, Any] | None = None,\n force_d_only: bool = False,\n provider: str = \"openai\",\n) -> dict[str, str]:\n \"\"\"\n Call the chosen provider (openai, xai, or deepseek) to translate a batch.\n\n The provider determines:\n - Which API endpoint and request shape is used (via build_request_body_for_provider)\n - How the response is parsed (via extract_text_from_provider_response)\n \"\"\"\n glossary = glossary or {}\n if force_d_only or all(item.get(\"field\") == \"d\" for item in batch):\n prompt = build_strong_d_only_prompt(glossary)\n else:\n prompt = (\n \"You are translating ArduPilot parameter metadata from English into Simplified Chinese \"\n \"for a flight-control ground station UI.\\n\"\n \"Rules:\\n\"\n \"1. Keep technical tokens unchanged: parameter names like YAW_RATE_ENABLE, MAVLink, GPS, RC, PWM, ACRO.\\n\"\n \"2. Use accurate avionics/flight-control terminology, not word-for-word machine translation.\\n\"\n \"3. Preserve paragraph breaks, list structure, numeric ranges, units, and punctuation intent.\\n\"\n \"4. The 'n' field should be short, natural UI text.\\n\"\n \"5. The 'd' field should be readable Chinese technical prose for pilots/engineers.\\n\"\n \"6. Do not add explanations, notes, or markdown.\\n\"\n \"7. Return one translation per input item.\\n\"\n )\n\n # When using DeepSeek with the simple \"json_object\" mode (no strict schema enforcement),\n # we must be extremely explicit. We also give a tiny concrete example of the exact shape.\n if provider == \"deepseek\":\n extra = (\n \"\\n\\nCRITICAL OUTPUT FORMAT (MANDATORY):\\n\"\n \"Your entire response MUST be exactly one JSON object and NOTHING ELSE.\\n\"\n \"Do not output any text before or after the JSON.\\n\"\n \"Do not use markdown code fences (```json ... ```).\\n\"\n \"The object must have exactly this shape:\\n\"\n ' {\"items\": [ {\"id\": \"d-1\", \"translation\": \"中文译文...\"}, {\"id\": \"d-2\", \"translation\": \"...\"} ] }\\n'\n \"Use the exact ids we gave you (d-1, d-2, ...). \"\n \"Only translate the 'source' text into clear Simplified Chinese. \"\n \"Output ONLY the raw JSON object described above.\"\n )\n prompt = prompt + extra\n\n input_payload = [\n {\n \"id\": f\"{item['field']}-{idx}\",\n \"field\": item[\"field\"],\n \"source\": item[\"source\"],\n }\n for idx, item in enumerate(batch, start=1)\n ]\n\n user_json_str = json.dumps({\"items\": input_payload}, ensure_ascii=False)\n schema = build_schema()\n\n body = build_request_body_for_provider(\n provider=provider,\n model=model,\n system_prompt=prompt,\n user_json_str=user_json_str,\n schema=schema,\n )\n\n if provider == \"deepseek\":\n api_url = DEEPSEEK_API_URL\n elif provider == \"xai\":\n api_url = XAI_API_URL\n else:\n api_url = OPENAI_API_URL\n\n request = urllib.request.Request(\n api_url,\n data=json.dumps(body).encode(\"utf-8\"),\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n method=\"POST\",\n )\n\n try:\n with urllib.request.urlopen(request, timeout=timeout_s) as response:\n response_json = json.loads(response.read().decode(\"utf-8\"))\n except urllib.error.HTTPError as exc:\n details = exc.read().decode(\"utf-8\", errors=\"replace\")\n if provider == \"deepseek\":\n prov_name = \"DeepSeek\"\n elif provider == \"xai\":\n prov_name = \"xAI\"\n else:\n prov_name = \"OpenAI\"\n raise RuntimeError(f\"{prov_name} API HTTP {exc.code}: {details}\") from exc\n\n raw_text = extract_text_from_provider_response(provider, response_json)\n\n try:\n parsed = normalize_translation_json(provider, raw_text)\n except Exception as exc:\n # For DeepSeek we want to show the user exactly what the model emitted\n if provider == \"deepseek\":\n preview = raw_text[:1500] if raw_text else \"(empty)\"\n raise RuntimeError(\n f\"DeepSeek returned something that could not be parsed as the expected JSON.\\n\"\n f\"Raw model output (first 1500 chars):\\n{preview}\\n\\n\"\n f\"Original error: {exc}\"\n ) from exc\n raise\n\n items = parsed.get(\"items\")\n if not isinstance(items, list):\n hint = \"\"\n if provider == \"deepseek\":\n preview = raw_text[:1200] if raw_text else \"(empty)\"\n hint = (\n f\"\\n\\nDeepSeek raw output (first 1200 chars) for debugging:\\n{preview}\\n\"\n \"The model did not follow the requested JSON shape. \"\n \"Common causes: it added explanations, used a different key name, \"\n \"or wrapped the result in extra text despite the prompt.\"\n )\n elif provider == \"xai\":\n hint = \" (xAI: try a current model such as grok-4.3 or grok-4.20-0309-reasoning that supports structured outputs)\"\n raise RuntimeError(f\"Structured output missing items array{hint}\")\n\n # Collect translations keyed by the id the model returned, while also\n # preserving order so we can fall back to positional matching when the\n # model echoes the wrong ids (a common DeepSeek json_object quirk).\n translations: dict[str, str] = {}\n ordered_translations: list[str] = []\n for item in items:\n if not isinstance(item, dict):\n continue\n item_id = item.get(\"id\")\n translation = item.get(\"translation\")\n if not isinstance(translation, str):\n continue\n translation = translation.strip()\n ordered_translations.append(translation)\n if isinstance(item_id, str):\n translations[item_id] = translation\n\n # Positional fallback: the caller looks up expected ids \"<field>-<idx>\"\n # (1-based, matching input_payload). If the model returned the right number\n # of items in order but with mismatched/renamed ids, recover by position.\n expected_ids = [f\"{item['field']}-{idx}\" for idx, item in enumerate(batch, start=1)]\n for pos, expected_id in enumerate(expected_ids):\n if translations.get(expected_id):\n continue\n if pos < len(ordered_translations) and ordered_translations[pos]:\n translations[expected_id] = ordered_translations[pos]\n\n return translations\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(\n description=\"Translate apm-param-db 'd' (description) field using OpenAI, xAI or DeepSeek. \"\n \"Use --review for safe, reviewable workflow (recommended).\"\n )\n parser.add_argument(\n \"--provider\",\n choices=[\"auto\", \"openai\", \"xai\", \"deepseek\"],\n default=\"auto\",\n help=\"AI provider. 'auto' (default) detects from key prefix (xai- → xAI). \"\n \"Use 'deepseek' for DeepSeek (requires explicit --provider deepseek; \"\n \"recommended with --api-key-env DEEPSEEK_API_KEY).\",\n )\n parser.add_argument(\n \"--model\",\n default=None,\n help=\"Model ID for the chosen provider (default: grok-4.3 for xAI, deepseek-v4-flash for DeepSeek, \"\n \"gpt-5.4-mini for OpenAI). For DeepSeek pass --provider deepseek and optionally --model.\",\n )\n parser.add_argument(\"--limit\", type=int, default=0, help=\"Maximum number of uncached strings to translate\")\n parser.add_argument(\"--batch-size\", type=int, default=DEFAULT_BATCH_SIZE, help=\"Strings per API call\")\n parser.add_argument(\"--sleep\", type=float, default=0.2, help=\"Sleep between API calls in seconds\")\n parser.add_argument(\n \"--max-retries\",\n type=int,\n default=3,\n help=\"Attempts per batch before giving up (handles flaky/incomplete API responses). Default: 3\",\n )\n parser.add_argument(\n \"--skip-failed\",\n action=\"store_true\",\n help=\"Skip a batch that is still incomplete after retries instead of aborting the run \"\n \"(missing items remain uncached and are retried on the next run).\",\n )\n parser.add_argument(\"--fields\", choices=[\"n\", \"d\", \"both\"], default=\"both\", help=\"Which fields to translate\")\n parser.add_argument(\n \"--review\",\n action=\"store_true\",\n help=\"Reviewable mode: generate editable review JSON files for 'd' only (strongly recommended for quality)\",\n )\n parser.add_argument(\"--review-dir\", default=str(REVIEW_DIR), help=\"Directory for review JSON files\")\n parser.add_argument(\"--timeout\", type=float, default=90.0, help=\"Per-request timeout in seconds\")\n parser.add_argument(\n \"--api-key-env\",\n default=\"OPENAI_API_KEY\",\n help=\"Environment variable holding the API key. For xAI prefer XAI_API_KEY and --api-key-env XAI_API_KEY\",\n )\n parser.add_argument(\"--cache-path\", default=str(AI_CACHE_PATH), help=\"Translation cache JSON path\")\n parser.add_argument(\"--source-path\", default=str(SOURCE_PATH), help=\"English source DB path\")\n parser.add_argument(\"--output-path\", default=str(DB_PATH), help=\"Translated output DB path\")\n parser.add_argument(\"--skip-backup\", action=\"store_true\", help=\"Do not write a pre-AI backup of the output file\")\n args = parser.parse_args()\n\n api_key = os.environ.get(args.api_key_env, \"\").strip()\n if not api_key:\n raise SystemExit(\n f\"Missing API key: set {args.api_key_env}\\n\"\n \" For DeepSeek: export DEEPSEEK_API_KEY=... then use --api-key-env DEEPSEEK_API_KEY --provider deepseek\\n\"\n \" For xAI: export XAI_API_KEY=... then use --api-key-env XAI_API_KEY\\n\"\n \" For OpenAI: export OPENAI_API_KEY=...\"\n )\n\n # Hardening against the very common \"copy-paste pollution\" problem on non-English systems.\n # API keys must be pure ASCII. Non-ASCII in the Authorization header causes the opaque\n # \"latin-1 codec can't encode\" crash deep inside urllib/http.client.\n if not api_key.isascii():\n # Show a short repr of the offending characters to help the user locate the garbage.\n bad = [(i, c, hex(ord(c))) for i, c in enumerate(api_key) if not c.isascii()]\n preview = api_key[:8] + \"...\" + api_key[-4:] if len(api_key) > 12 else api_key\n raise SystemExit(\n \"API key contains non-ASCII characters (very common after copy-paste from chat/browser).\\n\"\n f\" Variable : {args.api_key_env}\\n\"\n f\" Preview : {preview}\\n\"\n f\" Bad chars: {bad[:5]}{'...' if len(bad) > 5 else ''}\\n\\n\"\n \"Please re-export a clean key:\\n\"\n f\" export {args.api_key_env}='xai-XXXXXXXXXXXXXXXX...'\\n\"\n \" (use straight single quotes, paste ONLY the key, no extra characters, no Chinese punctuation)\\n\"\n \"Then re-run the command.\"\n )\n\n # Resolve provider (auto-detect from key prefix if --provider=auto)\n provider, _api_url, provider_default_model = resolve_provider(\n provider_arg=getattr(args, \"provider\", \"auto\"),\n api_key=api_key,\n )\n\n # Effective model: explicit --model wins, otherwise the provider's recommended default\n effective_model = args.model or provider_default_model\n\n source_path = Path(args.source_path)\n output_path = Path(args.output_path)\n cache_path = Path(args.cache_path)\n\n if not source_path.is_file():\n if source_path == SOURCE_PATH and DB_PATH.is_file():\n raise SystemExit(\n f\"Missing English source DB: {source_path}\\n\"\n f\"Current translated DB exists at {DB_PATH}. Please restore or provide --source-path.\"\n )\n raise SystemExit(f\"Missing source DB: {source_path}\")\n\n source_data = load_json(source_path)\n if not isinstance(source_data, dict):\n raise SystemExit(\"Source DB root is not an object\")\n\n cache: dict[str, str] = {}\n if cache_path.is_file():\n raw_cache = load_json(cache_path)\n if isinstance(raw_cache, dict):\n cache = {str(k): str(v) for k, v in raw_cache.items()}\n\n glossary = load_glossary()\n\n # In review mode we focus exclusively on 'd' (说明) as per project decision\n effective_fields = \"d\" if args.review else args.fields\n\n pending_all = []\n for item in collect_strings(source_data, effective_fields):\n cache_key = f\"{item['field']}\\t{item['source']}\"\n if cache_key not in cache:\n pending_all.append(item)\n\n pending = pending_all[: args.limit] if args.limit > 0 else pending_all\n\n print(\n f\"Provider: {provider} | model: {effective_model} | \"\n f\"Source entries: {len(source_data)} | uncached pending: {len(pending_all)}\"\n + (f\" | this run: {len(pending)}\" if args.limit > 0 else \"\")\n + (f\" | REVIEW MODE (d only)\" if args.review else \"\"),\n flush=True,\n )\n\n if not pending:\n print(\"Nothing to translate.\", flush=True)\n return\n\n done = 0\n review_items: list[dict[str, Any]] = []\n\n for start in range(0, len(pending), max(1, args.batch_size)):\n batch = pending[start : start + max(1, args.batch_size)]\n\n # Retry a flaky batch a few times before giving up. A single bad\n # response (e.g. DeepSeek returning mismatched ids or dropping items)\n # should not abort the whole run and lose hundreds of cached results.\n translations: dict[str, str] = {}\n missing: list[str] = []\n last_error: Exception | None = None\n for attempt in range(1, max(1, args.max_retries) + 1):\n try:\n translations = translate_batch(\n api_key=api_key,\n model=effective_model,\n batch=batch,\n timeout_s=args.timeout,\n glossary=glossary,\n force_d_only=args.review,\n provider=provider,\n )\n except Exception as exc: # transient API/parse failures\n last_error = exc\n translations = {}\n else:\n last_error = None\n\n missing = [\n f\"{item['field']}-{idx}\"\n for idx, item in enumerate(batch, start=1)\n if not translations.get(f\"{item['field']}-{idx}\", \"\").strip()\n ]\n if not missing and last_error is None:\n break\n\n if attempt < max(1, args.max_retries):\n backoff = args.sleep * (2 ** (attempt - 1)) + 0.5\n reason = (\n f\"error: {last_error}\" if last_error is not None\n else f\"incomplete ({len(missing)} missing)\"\n )\n print(\n f\" Batch at {start} {reason}; retry {attempt}/{args.max_retries - 1} \"\n f\"in {backoff:.1f}s\",\n flush=True,\n )\n time.sleep(backoff)\n\n if last_error is not None:\n raise RuntimeError(\n f\"Batch at offset {start} failed after {args.max_retries} attempts: {last_error}\"\n ) from last_error\n\n for idx, item in enumerate(batch, start=1):\n response_id = f\"{item['field']}-{idx}\"\n translated = translations.get(response_id, \"\").strip()\n if not translated:\n continue\n\n cache_key = f\"{item['field']}\\t{item['source']}\"\n cache[cache_key] = translated\n done += 1\n\n if args.review and item[\"field\"] == \"d\":\n # Try to enrich with param name (source_data keys are the param names)\n param_name = None\n for pname, pval in source_data.items():\n if isinstance(pval, dict) and pval.get(\"d\") == item[\"source\"]:\n param_name = pname\n break\n review_items.append({\n \"id\": response_id,\n \"param\": param_name or \"UNKNOWN\",\n \"field\": \"d\",\n \"source\": item[\"source\"],\n \"proposed\": translated,\n \"status\": \"pending\", # user should change to \"accepted\" or delete\n })\n\n if missing:\n msg = (\n f\"Batch at offset {start} still incomplete after {args.max_retries} attempts \"\n f\"({len(missing)} missing): {', '.join(missing)}\"\n )\n if args.skip_failed:\n print(f\" WARNING: {msg} — skipping (will be retried on next run).\", flush=True)\n else:\n raise RuntimeError(msg + \"\\nUse --skip-failed to continue past bad batches.\")\n\n save_json(cache_path, cache)\n print(f\"Translated {done}/{len(pending)}\", flush=True)\n time.sleep(max(0.0, args.sleep))\n\n review_dir = Path(args.review_dir)\n if args.review and review_items:\n review_path = write_review_file(review_items, review_dir)\n print(f\"\\nReview file written: {review_path}\", flush=True)\n print(\"Please review the 'proposed' translations, edit as needed, then run the apply tool.\", flush=True)\n print(\"No changes were written to the final db in review mode.\", flush=True)\n return\n\n # Normal (non-review) path — apply to output\n output_data = load_json(source_path)\n apply_cache(output_data, cache)\n\n if output_path.is_file() and not args.skip_backup:\n shutil.copy2(output_path, BACKUP_OUT_PATH)\n\n save_json(output_path, output_data)\n print(f\"Done. Updated {output_path} with {done} new AI translations.\", flush=True)\n\n\nif __name__ == \"__main__\":\n try:\n main()\n except KeyboardInterrupt:\n sys.exit(130)\n"}, "05767c1d4ac963c86ddc6774a2325836c27432bc": {"t": 1785763581.3984559, "n": 11961, "h": "587a6bc644d68192", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nTranslate ArduPilot parameter metadata with the OpenAI Responses API.\n\nThis script reads English source strings from JS/data/apm-param-db.en.json,\ntranslates them into Simplified Chinese in batches, caches results locally,\nand writes the merged output back to JS/data/apm-param-db.json.\n\nEnvironment:\n OPENAI_API_KEY Required.\n\nExample:\n python translate_params_ai.py --model gpt-5.4-mini --limit 200\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport os\nimport shutil\nimport sys\nimport time\nimport urllib.error\nimport urllib.request\nfrom pathlib import Path\nfrom typing import Any\n\nROOT = Path(__file__).resolve().parent\nDB_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.json\"\nSOURCE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.en.json\"\nLEGACY_CACHE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-translate-cache.json\"\nAI_CACHE_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-translate-cache.ai.json\"\nBACKUP_OUT_PATH = ROOT / \"JS\" / \"data\" / \"apm-param-db.pre-ai-backup.json\"\n\nDEFAULT_MODEL = \"gpt-5.4-mini\"\nDEFAULT_BATCH_SIZE = 24\nAPI_URL = \"https://api.openai.com/v1/responses\"\n\n\ndef zh_ratio(text: str) -> float:\n if not text:\n return 0.0\n zh = sum(1 for ch in text if \"\\u4e00\" <= ch <= \"\\u9fff\")\n return zh / len(text)\n\n\ndef needs_translation(text: str, *, for_description: bool) -> bool:\n text = (text or \"\").strip()\n if not text:\n return False\n ratio = zh_ratio(text)\n if for_description:\n return len(text) > 8 and ratio < 0.18\n return ratio < 0.08\n\n\ndef load_json(path: Path) -> dict[str, Any]:\n return json.loads(path.read_text(encoding=\"utf-8\"))\n\n\ndef save_json(path: Path, obj: object) -> None:\n path.write_text(json.dumps(obj, ensure_ascii=False, indent=2), encoding=\"utf-8\")\n\n\ndef collect_strings(data: dict[str, Any], fields: str) -> list[dict[str, str]]:\n collected: list[dict[str, str]] = []\n seen: set[tuple[str, str]] = set()\n include_n = fields in {\"n\", \"both\"}\n include_d = fields in {\"d\", \"both\"}\n\n for value in data.values():\n if not isinstance(value, dict):\n continue\n if include_n:\n name = value.get(\"n\")\n if isinstance(name, str) and needs_translation(name, for_description=False):\n key = (\"n\", name.strip())\n if key not in seen:\n seen.add(key)\n collected.append({\"field\": \"n\", \"source\": name.strip()})\n if include_d:\n desc = value.get(\"d\")\n if isinstance(desc, str) and needs_translation(desc, for_description=True):\n key = (\"d\", desc.strip())\n if key not in seen:\n seen.add(key)\n collected.append({\"field\": \"d\", \"source\": desc.strip()})\n return collected\n\n\ndef apply_cache(data: dict[str, Any], cache: dict[str, str]) -> None:\n for value in data.values():\n if not isinstance(value, dict):\n continue\n for field in (\"n\", \"d\"):\n src = value.get(field)\n if isinstance(src, str):\n key = f\"{field}\\t{src.strip()}\"\n if key in cache:\n value[field] = cache[key]\n\n\ndef build_schema() -> dict[str, Any]:\n return {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"items\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"id\": {\"type\": \"string\"},\n \"translation\": {\"type\": \"string\"},\n },\n \"required\": [\"id\", \"translation\"],\n },\n }\n },\n \"required\": [\"items\"],\n }\n\n\ndef extract_output_text(response_json: dict[str, Any]) -> str:\n output_text = response_json.get(\"output_text\")\n if isinstance(output_text, str) and output_text.strip():\n return output_text\n\n output = response_json.get(\"output\")\n if isinstance(output, list):\n chunks: list[str] = []\n for item in output:\n if not isinstance(item, dict):\n continue\n content = item.get(\"content\")\n if not isinstance(content, list):\n continue\n for part in content:\n if not isinstance(part, dict):\n continue\n text = part.get(\"text\")\n if isinstance(text, str):\n chunks.append(text)\n if chunks:\n return \"\\n\".join(chunks)\n raise RuntimeError(\"Responses API returned no readable text output\")\n\n\ndef translate_batch(\n *,\n api_key: str,\n model: str,\n batch: list[dict[str, str]],\n timeout_s: float,\n) -> dict[str, str]:\n prompt = (\n \"You are translating ArduPilot parameter metadata from English into Simplified Chinese \"\n \"for a flight-control ground station UI.\\n\"\n \"Rules:\\n\"\n \"1. Keep technical tokens unchanged: parameter names like YAW_RATE_ENABLE, MAVLink, GPS, RC, PWM, ACRO.\\n\"\n \"2. Use accurate avionics/flight-control terminology, not word-for-word machine translation.\\n\"\n \"3. Preserve paragraph breaks, list structure, numeric ranges, units, and punctuation intent.\\n\"\n \"4. The 'n' field should be short, natural UI text.\\n\"\n \"5. The 'd' field should be readable Chinese technical prose for pilots/engineers.\\n\"\n \"6. Do not add explanations, notes, or markdown.\\n\"\n \"7. Return one translation per input item.\\n\"\n )\n\n input_payload = [\n {\n \"id\": f\"{item['field']}-{idx}\",\n \"field\": item[\"field\"],\n \"source\": item[\"source\"],\n }\n for idx, item in enumerate(batch, start=1)\n ]\n\n body = {\n \"model\": model,\n \"input\": [\n {\n \"role\": \"system\",\n \"content\": [{\"type\": \"input_text\", \"text\": prompt}],\n },\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": json.dumps({\"items\": input_payload}, ensure_ascii=False),\n }\n ],\n },\n ],\n \"text\": {\n \"format\": {\n \"type\": \"json_schema\",\n \"name\": \"param_translation_batch\",\n \"schema\": build_schema(),\n \"strict\": True,\n }\n },\n }\n\n request = urllib.request.Request(\n API_URL,\n data=json.dumps(body).encode(\"utf-8\"),\n headers={\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n },\n method=\"POST\",\n )\n\n try:\n with urllib.request.urlopen(request, timeout=timeout_s) as response:\n response_json = json.loads(response.read().decode(\"utf-8\"))\n except urllib.error.HTTPError as exc:\n details = exc.read().decode(\"utf-8\", errors=\"replace\")\n raise RuntimeError(f\"OpenAI API HTTP {exc.code}: {details}\") from exc\n\n text = extract_output_text(response_json)\n parsed = json.loads(text)\n items = parsed.get(\"items\")\n if not isinstance(items, list):\n raise RuntimeError(\"Structured output missing items array\")\n\n translations: dict[str, str] = {}\n for item in items:\n if not isinstance(item, dict):\n continue\n item_id = item.get(\"id\")\n translation = item.get(\"translation\")\n if isinstance(item_id, str) and isinstance(translation, str):\n translations[item_id] = translation.strip()\n return translations\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(description=\"Translate apm-param-db with the OpenAI Responses API\")\n parser.add_argument(\"--model\", default=DEFAULT_MODEL, help=f\"Responses API model id (default: {DEFAULT_MODEL})\")\n parser.add_argument(\"--limit\", type=int, default=0, help=\"Maximum number of uncached strings to translate\")\n parser.add_argument(\"--batch-size\", type=int, default=DEFAULT_BATCH_SIZE, help=\"Strings per API call\")\n parser.add_argument(\"--sleep\", type=float, default=0.2, help=\"Sleep between API calls in seconds\")\n parser.add_argument(\"--fields\", choices=[\"n\", \"d\", \"both\"], default=\"both\", help=\"Which fields to translate\")\n parser.add_argument(\"--timeout\", type=float, default=90.0, help=\"Per-request timeout in seconds\")\n parser.add_argument(\"--api-key-env\", default=\"OPENAI_API_KEY\", help=\"Environment variable that stores the API key\")\n parser.add_argument(\"--cache-path\", default=str(AI_CACHE_PATH), help=\"Translation cache JSON path\")\n parser.add_argument(\"--source-path\", default=str(SOURCE_PATH), help=\"English source DB path\")\n parser.add_argument(\"--output-path\", default=str(DB_PATH), help=\"Translated output DB path\")\n parser.add_argument(\"--skip-backup\", action=\"store_true\", help=\"Do not write a pre-AI backup of the output file\")\n args = parser.parse_args()\n\n api_key = os.environ.get(args.api_key_env, \"\").strip()\n if not api_key:\n raise SystemExit(f\"Missing API key: set {args.api_key_env}\")\n\n source_path = Path(args.source_path)\n output_path = Path(args.output_path)\n cache_path = Path(args.cache_path)\n\n if not source_path.is_file():\n if source_path == SOURCE_PATH and DB_PATH.is_file():\n raise SystemExit(\n f\"Missing English source DB: {source_path}\\n\"\n f\"Current translated DB exists at {DB_PATH}. Please restore or provide --source-path.\"\n )\n raise SystemExit(f\"Missing source DB: {source_path}\")\n\n source_data = load_json(source_path)\n if not isinstance(source_data, dict):\n raise SystemExit(\"Source DB root is not an object\")\n\n cache: dict[str, str] = {}\n if cache_path.is_file():\n raw_cache = load_json(cache_path)\n if isinstance(raw_cache, dict):\n cache = {str(k): str(v) for k, v in raw_cache.items()}\n\n pending_all = []\n for item in collect_strings(source_data, args.fields):\n cache_key = f\"{item['field']}\\t{item['source']}\"\n if cache_key not in cache:\n pending_all.append(item)\n\n pending = pending_all[: args.limit] if args.limit > 0 else pending_all\n\n print(\n f\"Source entries: {len(source_data)} | uncached pending: {len(pending_all)}\"\n + (f\" | this run: {len(pending)}\" if args.limit > 0 else \"\"),\n flush=True,\n )\n\n done = 0\n for start in range(0, len(pending), max(1, args.batch_size)):\n batch = pending[start : start + max(1, args.batch_size)]\n translations = translate_batch(\n api_key=api_key,\n model=args.model,\n batch=batch,\n timeout_s=args.timeout,\n )\n\n missing: list[str] = []\n for idx, item in enumerate(batch, start=1):\n response_id = f\"{item['field']}-{idx}\"\n translated = translations.get(response_id, \"\").strip()\n if not translated:\n missing.append(response_id)\n continue\n cache[f\"{item['field']}\\t{item['source']}\"] = translated\n done += 1\n\n if missing:\n raise RuntimeError(f\"Batch returned incomplete translations: {', '.join(missing)}\")\n\n save_json(cache_path, cache)\n print(f\"Translated {done}/{len(pending)}\", flush=True)\n time.sleep(max(0.0, args.sleep))\n\n output_data = load_json(source_path)\n apply_cache(output_data, cache)\n\n if output_path.is_file() and not args.skip_backup:\n shutil.copy2(output_path, BACKUP_OUT_PATH)\n\n save_json(output_path, output_data)\n print(f\"Done. Updated {output_path} with {done} new AI translations.\", flush=True)\n\n\nif __name__ == \"__main__\":\n try:\n main()\n except KeyboardInterrupt:\n sys.exit(130)\n"}}}} |