Files
hack/tools/scripts/llm-key-hunter/results/_cache/content/caiyilian_anime-accurate-sub.json
T

1 line
102 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{"repo": "caiyilian/anime-accurate-sub", "files": {"scripts/season_monitor.py": {"4d507921cd926e659e5a5a22d58eec34686302b8": {"t": 1785768246.6065085, "n": 21253, "h": "c8d3c05e16668cd8", "c": "\"\"\"Read-only progress collector and dashboard for a full-season pipeline run.\"\"\"\n\n# The embedded, self-contained dashboard intentionally keeps CSS and JS compact.\n# ruff: noqa: E501\n\nfrom __future__ import annotations\n\nimport json\nimport re\nimport time\nfrom collections import Counter\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\nSEASON_STAGES = (\n (\"japanese_subtitle\", \"日文字幕\", 2),\n (\"translate\", \"Sakura 翻译\", 10),\n (\"multi_agent_review\", \"多 Agent 审查\", 38),\n (\"mqm_quality_review\", \"双裁判 MQM\", 42),\n (\"subtitle\", \"生成字幕\", 1),\n (\"embed_subtitle\", \"压制视频\", 6),\n (\"quality_check\", \"规则质检\", 1),\n)\nSTAGE_LABELS = dict((stage, label) for stage, label, _ in SEASON_STAGES)\nSTAGE_WEIGHTS = dict((stage, weight) for stage, _, weight in SEASON_STAGES)\nREVIEW_FILES = {\n \"multi_agent_review\": (\"multi_agent_review.progress.jsonl\", \"translated.json\"),\n \"mqm_quality_review\": (\"mqm_quality_review.progress.jsonl\", \"reviewed.json\"),\n}\nEPISODE_PATTERN = re.compile(r\"第\\s*(\\d{1,3})\\s*集\")\nANSI_PATTERN = re.compile(r\"\\x1b\\[[0-?]*[ -/]*[@-~]\")\nSECRET_PATTERNS = (\n re.compile(r\"(?i)(authorization\\s*[:=]\\s*bearer\\s+)[^\\s,;]+\"),\n re.compile(r\"(?i)((?:api[_-]?key|token|secret)\\s*[:=]\\s*)[^\\s,;]+\"),\n re.compile(r\"\\bsk-[A-Za-z0-9_-]{8,}\\b\"),\n)\n\n\ndef _iso_timestamp(timestamp: float | None) -> str | None:\n if timestamp is None:\n return None\n return datetime.fromtimestamp(timestamp).astimezone().isoformat(timespec=\"seconds\")\n\n\ndef _read_json(path: Path) -> Any:\n try:\n return json.loads(path.read_text(encoding=\"utf-8\"))\n except (OSError, UnicodeError, json.JSONDecodeError):\n return None\n\n\ndef _redact_log(text: str) -> str:\n redacted = ANSI_PATTERN.sub(\"\", text)\n for pattern in SECRET_PATTERNS:\n if pattern.groups:\n redacted = pattern.sub(r\"\\1[已隐藏]\", redacted)\n else:\n redacted = pattern.sub(\"[已隐藏]\", redacted)\n return redacted\n\n\nclass SeasonProgressMonitor:\n \"\"\"Summarize durable pipeline artifacts without controlling the process.\"\"\"\n\n def __init__(\n self,\n root: str | Path,\n episode_count: int = 14,\n stale_after_seconds: int = 300,\n log_line_count: int = 36,\n ) -> None:\n self.root = Path(root).resolve()\n self.episode_count = max(1, int(episode_count))\n self.stale_after_seconds = max(30, int(stale_after_seconds))\n self.log_line_count = max(5, int(log_line_count))\n self._jsonl_cache: dict[Path, tuple[tuple[int, int], dict[str, Any]]] = {}\n self._count_cache: dict[Path, tuple[tuple[int, int], int]] = {}\n\n @staticmethod\n def _signature(path: Path) -> tuple[int, int] | None:\n try:\n stat = path.stat()\n return stat.st_size, stat.st_mtime_ns\n except OSError:\n return None\n\n def _segment_count(self, path: Path) -> int:\n signature = self._signature(path)\n if signature is None:\n return 0\n cached = self._count_cache.get(path)\n if cached and cached[0] == signature:\n return cached[1]\n payload = _read_json(path)\n count = len(payload) if isinstance(payload, list) else 0\n self._count_cache[path] = (signature, count)\n return count\n\n def _review_progress(self, path: Path, total: int) -> dict[str, Any]:\n signature = self._signature(path)\n empty = {\"processed\": 0, \"total\": total, \"percent\": 0, \"statuses\": {}}\n if signature is None:\n return empty\n cached = self._jsonl_cache.get(path)\n if cached and cached[0] == signature:\n summary = dict(cached[1])\n summary[\"total\"] = total\n summary[\"percent\"] = round(100 * summary[\"processed\"] / total, 1) if total else 0\n return summary\n\n records: dict[int, dict[str, Any]] = {}\n try:\n with path.open(\"r\", encoding=\"utf-8\", errors=\"replace\") as source:\n for line in source:\n try:\n row = json.loads(line)\n except json.JSONDecodeError:\n continue\n position = row.get(\"position\") if isinstance(row, dict) else None\n if isinstance(position, int) and position >= 0:\n records[position] = row\n except OSError:\n return empty\n\n statuses = Counter(str(row.get(\"status\") or \"unknown\") for row in records.values())\n processed = len(records)\n summary = {\n \"processed\": processed,\n \"total\": total,\n \"percent\": round(100 * processed / total, 1) if total else 0,\n \"statuses\": dict(sorted(statuses.items())),\n }\n self._jsonl_cache[path] = (signature, dict(summary))\n return summary\n\n def _episode_directories(self) -> dict[int, Path]:\n result: dict[int, Path] = {}\n if not self.root.is_dir():\n return result\n try:\n children = self.root.iterdir()\n except OSError:\n return result\n for path in children:\n if not path.is_dir():\n continue\n match = EPISODE_PATTERN.search(path.name)\n if match:\n result[int(match.group(1))] = path\n return result\n\n @staticmethod\n def _newest_mtime(path: Path | None) -> float | None:\n if path is None or not path.is_dir():\n return None\n newest: float | None = None\n try:\n for child in path.iterdir():\n if child.is_file():\n modified = child.stat().st_mtime\n newest = modified if newest is None else max(newest, modified)\n except OSError:\n return newest\n return newest\n\n def _quality_summary(self, episode_dir: Path | None) -> dict[str, Any]:\n if episode_dir is None:\n return {}\n result: dict[str, Any] = {}\n mqm = _read_json(episode_dir / \"mqm_quality_report.json\")\n if isinstance(mqm, dict) and isinstance(mqm.get(\"summary\"), dict):\n result[\"mqm\"] = {\n key: mqm[\"summary\"].get(key, 0)\n for key in (\"approved\", \"corrected\", \"needs_review\", \"errors\", \"applied\")\n }\n final = _read_json(episode_dir / \"final_adjudication_summary.json\")\n if isinstance(final, dict) and isinstance(final.get(\"summary\"), dict):\n final_summary = final[\"summary\"]\n resolved = min(\n int(result[\"mqm\"][\"needs_review\"]),\n int(final_summary.get(\"resolved_needs_review\", 0)),\n )\n kept = min(resolved, int(final_summary.get(\"needs_review_kept\", 0)))\n revised = min(\n resolved - kept,\n int(final_summary.get(\"needs_review_revised\", 0)),\n )\n result[\"mqm\"][\"approved\"] += kept\n result[\"mqm\"][\"corrected\"] += revised\n result[\"mqm\"][\"needs_review\"] -= resolved\n result[\"mqm\"][\"final_reviewed\"] = int(\n final_summary.get(\"reviewed\", resolved)\n )\n quality = _read_json(episode_dir / \"quality_report.json\")\n if isinstance(quality, dict) and isinstance(quality.get(\"stats\"), dict):\n result[\"rules\"] = {\n key: quality[\"stats\"].get(key, 0)\n for key in (\"total_issues\", \"errors\", \"warnings\", \"info\")\n }\n return result\n\n def _episode_snapshot(self, number: int, episode_dir: Path | None) -> dict[str, Any]:\n checkpoint = _read_json(episode_dir / \"checkpoint.json\") if episode_dir else None\n checkpoint = checkpoint if isinstance(checkpoint, dict) else {}\n completed = {\n stage for stage, _, _ in SEASON_STAGES\n if isinstance(checkpoint.get(stage), dict)\n and checkpoint[stage].get(\"status\") == \"completed\"\n }\n failed = {\n stage for stage, _, _ in SEASON_STAGES\n if isinstance(checkpoint.get(stage), dict)\n and checkpoint[stage].get(\"status\") == \"failed\"\n }\n active = next((stage for stage, _, _ in SEASON_STAGES if stage not in completed), None)\n review = {\"processed\": 0, \"total\": 0, \"percent\": 0, \"statuses\": {}}\n if active in REVIEW_FILES and episode_dir:\n progress_name, source_name = REVIEW_FILES[active]\n total = self._segment_count(episode_dir / source_name)\n review = self._review_progress(episode_dir / progress_name, total)\n\n stages = []\n weighted_progress = 0.0\n for stage, label, weight in SEASON_STAGES:\n if stage in completed:\n state, fraction = \"completed\", 1.0\n elif stage in failed:\n state, fraction = \"failed\", 0.0\n elif stage == active:\n state = \"active\"\n fraction = review[\"percent\"] / 100 if stage in REVIEW_FILES else 0.0\n else:\n state, fraction = \"pending\", 0.0\n weighted_progress += weight * fraction\n stages.append({\"id\": stage, \"label\": label, \"state\": state})\n\n if len(completed) == len(SEASON_STAGES):\n state = \"completed\"\n elif failed:\n state = \"failed\"\n elif checkpoint or review[\"processed\"]:\n state = \"running\"\n else:\n state = \"pending\"\n updated = self._newest_mtime(episode_dir)\n return {\n \"number\": number,\n \"name\": episode_dir.name if episode_dir else f\"第{number:02d}集\",\n \"state\": state,\n \"percent\": round(weighted_progress, 1),\n \"active_stage\": active,\n \"active_stage_label\": STAGE_LABELS.get(active, \"已完成\"),\n \"completed_stages\": len(completed),\n \"total_stages\": len(SEASON_STAGES),\n \"stages\": stages,\n \"review\": review,\n \"quality\": self._quality_summary(episode_dir),\n \"updated_at\": _iso_timestamp(updated),\n \"_updated_epoch\": updated,\n }\n\n def _log_tail(self) -> tuple[str, float | None]:\n paths = [self.root / \"full_run_v3.stdout.log\", self.root / \"full_run.stdout.log\"]\n path = next((candidate for candidate in paths if candidate.is_file()), None)\n if path is None:\n return \"\", None\n try:\n size = path.stat().st_size\n with path.open(\"rb\") as source:\n source.seek(max(0, size - 65_536))\n lines = source.read().decode(\"utf-8\", errors=\"replace\").splitlines()\n return _redact_log(\"\\n\".join(lines[-self.log_line_count :])), path.stat().st_mtime\n except OSError:\n return \"\", None\n\n def snapshot(self) -> dict[str, Any]:\n directories = self._episode_directories()\n episodes = [\n self._episode_snapshot(number, directories.get(number))\n for number in range(1, self.episode_count + 1)\n ]\n log_tail, log_mtime = self._log_tail()\n activity_times = [\n timestamp for timestamp in [log_mtime, *(item[\"_updated_epoch\"] for item in episodes)]\n if timestamp is not None\n ]\n last_activity = max(activity_times) if activity_times else None\n now = time.time()\n completed_episodes = sum(item[\"state\"] == \"completed\" for item in episodes)\n failed = any(item[\"state\"] == \"failed\" for item in episodes)\n if completed_episodes == self.episode_count:\n status = \"completed\"\n elif failed:\n status = \"failed\"\n elif last_activity is not None and now - last_activity <= self.stale_after_seconds:\n status = \"running\"\n elif last_activity is not None:\n status = \"stale\"\n else:\n status = \"waiting\"\n\n active_episode = next((item for item in episodes if item[\"state\"] == \"running\"), None)\n if active_episode is None:\n active_episode = next((item for item in episodes if item[\"state\"] != \"completed\"), None)\n overall = sum(item[\"percent\"] for item in episodes) / self.episode_count\n for episode in episodes:\n episode.pop(\"_updated_epoch\", None)\n return {\n \"status\": status,\n \"status_label\": {\n \"running\": \"运行正常\",\n \"completed\": \"全季完成\",\n \"failed\": \"发现失败阶段\",\n \"stale\": \"等待新进度\",\n \"waiting\": \"尚未开始\",\n }[status],\n \"overall_percent\": round(overall, 1),\n \"completed_episodes\": completed_episodes,\n \"episode_count\": self.episode_count,\n \"active_episode\": active_episode[\"number\"] if active_episode else None,\n \"last_activity_at\": _iso_timestamp(last_activity),\n \"captured_at\": _iso_timestamp(now),\n \"root_name\": self.root.name,\n \"episodes\": episodes,\n \"log_tail\": log_tail,\n }\n\n\nMONITOR_HTML = \"\"\"<!doctype html>\n<html lang=\"zh-CN\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\n<title>轻音少女全季进度 · Anime Accurate Sub</title>\n<style>\n:root{color-scheme:dark;--ink:#f7f5f2;--muted:#a9b0bd;--bg:#090c12;--panel:#111620;--line:#263040;--pink:#ff6fae;--mint:#55d6be;--amber:#ffca72;--red:#ff7272}*{box-sizing:border-box}body{margin:0;color:var(--ink);background:radial-gradient(circle at 18% -10%,#34213a 0,transparent 34%),radial-gradient(circle at 92% 4%,#153b3a 0,transparent 28%),var(--bg);font:15px/1.5 Inter,\"Microsoft YaHei UI\",system-ui,sans-serif}main{width:min(1220px,94vw);margin:0 auto;padding:42px 0 70px}.top{display:flex;justify-content:space-between;gap:24px;align-items:flex-start}.eyebrow{color:var(--pink);font-size:12px;letter-spacing:.18em;text-transform:uppercase;font-weight:800}h1{font-size:clamp(30px,5vw,56px);line-height:1.08;margin:8px 0 10px;letter-spacing:-.035em}.sub{margin:0;color:var(--muted);max-width:670px}.live{display:flex;align-items:center;gap:9px;padding:10px 14px;border:1px solid var(--line);border-radius:99px;background:#111620cc;white-space:nowrap}.live i{width:9px;height:9px;border-radius:50%;background:var(--muted)}.live.running i{background:var(--mint);box-shadow:0 0 0 6px #55d6be20;animation:pulse 1.8s infinite}.live.failed i{background:var(--red)}.live.stale i{background:var(--amber)}@keyframes pulse{50%{box-shadow:0 0 0 11px #55d6be00}}.summary{display:grid;grid-template-columns:230px 1fr;gap:18px;margin:30px 0}.panel{background:linear-gradient(150deg,#151b27e8,#0e131ce8);border:1px solid var(--line);border-radius:20px;box-shadow:0 24px 65px #0005}.ring-card{display:grid;place-items:center;padding:25px}.ring{--p:0;width:154px;aspect-ratio:1;border-radius:50%;display:grid;place-items:center;background:radial-gradient(circle closest-side,#111620 76%,transparent 77%),conic-gradient(var(--pink) calc(var(--p)*1%),#273040 0)}.ring strong{font-size:31px}.ring span{display:block;color:var(--muted);font-size:11px;text-align:center}.now{padding:24px 26px}.now-head{display:flex;justify-content:space-between;gap:16px}.now h2{margin:3px 0 4px;font-size:23px}.meta{color:var(--muted);font-size:13px}.review-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin-top:20px}.stat{background:#0a0e15;border:1px solid #202938;border-radius:12px;padding:12px}.stat b{font-size:21px;display:block}.stat span{color:var(--muted);font-size:12px}.section-title{display:flex;align-items:end;justify-content:space-between;margin:35px 0 13px}.section-title h2{margin:0}.episodes{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px}.episode{padding:17px 18px;transition:border-color .2s,transform .2s}.episode.active{border-color:#ff6fae88;transform:translateY(-1px)}.ep-head{display:flex;align-items:center;justify-content:space-between;gap:15px}.ep-name{font-weight:750}.ep-pct{font-variant-numeric:tabular-nums;color:var(--muted)}.bar{height:7px;border-radius:9px;background:#242d3b;overflow:hidden;margin:11px 0}.bar i{display:block;height:100%;background:linear-gradient(90deg,var(--pink),#a589ff,var(--mint));border-radius:inherit}.stages{display:grid;grid-template-columns:repeat(7,1fr);gap:5px}.stage{height:6px;border-radius:9px;background:#2b3442}.stage.completed{background:var(--mint)}.stage.active{background:var(--pink);box-shadow:0 0 9px #ff6fae88}.stage.failed{background:var(--red)}.ep-detail{display:flex;justify-content:space-between;gap:10px;margin-top:9px;color:var(--muted);font-size:12px}.terminal{margin-top:13px;padding:18px}.terminal pre{margin:12px 0 0;padding:16px;max-height:350px;overflow:auto;border-radius:12px;background:#070a0f;color:#bdc9d8;font:12px/1.65 ui-monospace,Consolas,monospace;white-space:pre-wrap}.empty{color:var(--muted)}.error{padding:24px;color:#ffd0d0}.foot{margin-top:13px;color:var(--muted);font-size:12px;text-align:right}@media(max-width:780px){main{padding-top:25px}.top{display:block}.live{margin-top:18px;width:max-content}.summary{grid-template-columns:1fr}.ring-card{padding:20px}.ring{width:130px}.episodes{grid-template-columns:1fr}.review-grid{grid-template-columns:1fr 1fr}.now-head{display:block}}\n</style></head><body><main>\n<header class=\"top\"><div><div class=\"eyebrow\">Anime Accurate Sub · Quality Run</div><h1>轻音少女 S1<br>全流程监控</h1><p class=\"sub\">Sakura 翻译、SenseNova 多 Agent 审查与双裁判 MQM 的实时执行状态。页面每 4 秒自动读取持久化进度。</p></div><div id=\"live\" class=\"live\"><i></i><span>正在连接</span></div></header>\n<section id=\"content\" aria-live=\"polite\"><div class=\"panel error\">正在读取全季进度……</div></section>\n<div id=\"foot\" class=\"foot\"></div>\n</main><script>\nconst content=document.querySelector('#content'),live=document.querySelector('#live'),foot=document.querySelector('#foot');\nconst esc=s=>String(s??'').replace(/[&<>\\\"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','\\\"':'&quot;'}[c]));\nconst n=v=>Number.isFinite(Number(v))?Number(v):0;\nfunction timeLabel(value){if(!value)return '暂无';try{return new Intl.DateTimeFormat('zh-CN',{month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit',second:'2-digit',hour12:false}).format(new Date(value))}catch{return value}}\nfunction render(data){live.className='live '+esc(data.status);live.innerHTML=`<i></i><span>${esc(data.status_label)}</span>`;const current=data.episodes.find(x=>x.number===data.active_episode);const r=current?.review||{},s=r.statuses||{};const currentBody=current?`<div class=\"now-head\"><div><div class=\"eyebrow\">当前任务 · 第 ${String(current.number).padStart(2,'0')} 集</div><h2>${esc(current.active_stage_label)}</h2><div class=\"meta\">${r.total?`已审查 ${n(r.processed)} / ${n(r.total)} 句 · ${n(r.percent).toFixed(1)}%`:`已完成 ${n(current.completed_stages)} / ${n(current.total_stages)} 个阶段`}</div></div><div class=\"meta\">最近写入 ${timeLabel(current.updated_at)}</div></div><div class=\"review-grid\"><div class=\"stat\"><b>${n(s.approved)}</b><span>审查通过</span></div><div class=\"stat\"><b>${n(s.corrected)}</b><span>自动修正</span></div><div class=\"stat\"><b>${n(s.needs_review)}</b><span>待人工复核</span></div><div class=\"stat\"><b>${n(s.error)+n(s.errors)}</b><span>调用/解析异常</span></div></div>`:'<div class=\"empty\">全部集数已处理完成。</div>';\nconst cards=data.episodes.map(ep=>{const review=ep.review||{},detail=review.total?`${n(review.processed)} / ${n(review.total)} 句`:`${n(ep.completed_stages)} / ${n(ep.total_stages)} 阶段`;return `<article class=\"panel episode ${ep.number===data.active_episode?'active':''}\"><div class=\"ep-head\"><span class=\"ep-name\">第 ${String(ep.number).padStart(2,'0')} 集</span><span class=\"ep-pct\">${n(ep.percent).toFixed(1)}%</span></div><div class=\"bar\"><i style=\"width:${Math.max(0,Math.min(100,n(ep.percent)))}%\"></i></div><div class=\"stages\" aria-label=\"七阶段进度\">${ep.stages.map(stage=>`<i class=\"stage ${esc(stage.state)}\" title=\"${esc(stage.label)}:${esc(stage.state)}\"></i>`).join('')}</div><div class=\"ep-detail\"><span>${esc(ep.state==='completed'?'全部完成':ep.active_stage_label)}</span><span>${detail}</span></div></article>`}).join('');\ncontent.innerHTML=`<section class=\"summary\"><div class=\"panel ring-card\"><div class=\"ring\" style=\"--p:${n(data.overall_percent)}\"><div><strong>${n(data.overall_percent).toFixed(1)}%</strong><span>全季质量流程</span></div></div></div><div class=\"panel now\">${currentBody}</div></section><div class=\"section-title\"><h2>${n(data.episode_count)} 集执行矩阵</h2><span class=\"meta\">已完整交付 ${n(data.completed_episodes)} / ${n(data.episode_count)} 集</span></div><section class=\"episodes\">${cards}</section><div class=\"section-title\"><h2>实时日志</h2><span class=\"meta\">最后心跳 ${timeLabel(data.last_activity_at)}</span></div><section class=\"panel terminal\"><pre>${esc(data.log_tail)||'暂时没有日志输出'}</pre></section>`;foot.textContent=`数据刷新于 ${timeLabel(data.captured_at)} · 只读监控,不会控制流水线`}\nasync function refresh(){try{const response=await fetch('/api/monitor',{cache:'no-store'});if(!response.ok)throw new Error(response.statusText);render(await response.json())}catch(error){live.className='live failed';live.innerHTML='<i></i><span>读取失败</span>';content.innerHTML=`<div class=\"panel error\">暂时无法读取进度:${esc(error.message)}。页面会自动重试。</div>`}}\nrefresh();setInterval(refresh,4000);\n</script></body></html>\"\"\"\n"}}, "config/quality_review.sensenova.json": {"4d507921cd926e659e5a5a22d58eec34686302b8": {"t": 1785768255.8765476, "n": 856, "h": "4e5a517e9ae23875", "c": "{\n \"provider\": \"openai\",\n \"base_url\": \"https://token.sensenova.cn/v1\",\n \"api_key_file\": \"config/sensenova_apikeys\",\n \"review_model\": \"sensenova-6.7-flash-lite\",\n \"review_fallback_model\": \"deepseek-v4-flash\",\n \"agent_models\": {\n \"consistency\": \"sensenova-6.7-flash-lite\",\n \"naturalness\": \"sensenova-6.7-flash-lite\",\n \"accuracy\": \"deepseek-v4-flash\",\n \"asr_check\": \"sensenova-6.7-flash-lite\",\n \"style\": \"sensenova-6.7-flash-lite\"\n },\n \"editor_provider\": \"openai\",\n \"editor_base_url\": \"https://token.sensenova.cn/v1\",\n \"editor_api_key_file\": \"config/sensenova_apikeys\",\n \"editor_model\": \"sensenova-6.7-flash-lite\",\n \"editor_fallback_model\": \"deepseek-v4-flash\",\n \"max_workers\": 2,\n \"context_window\": 5,\n \"min_fix_votes\": 2,\n \"min_reviewer_confidence\": 0.75,\n \"min_editor_confidence\": 0.85,\n \"retries\": 6,\n \"timeout_s\": 600\n}\n"}}, "config/translator.sakura-remote.example.json": {"4d507921cd926e659e5a5a22d58eec34686302b8": {"t": 1785768255.9559946, "n": 997, "h": "7f4907ead49e90fa", "c": "{\n \"backend\": \"sakura\",\n \"sakura\": {\n \"model\": \"crosery/sakura-14b-qwen2.5-v1.0-q6k:latest\",\n \"host\": \"172.31.102.189\",\n \"timeout_s\": 300,\n \"max_retries\": 4,\n \"validation_retries\": 2,\n \"validation_fallback_backend\": \"galtransl\",\n \"validation_rescue_models\": [\n \"sensenova-6.7-flash-lite\",\n \"deepseek-v4-flash\"\n ],\n \"validation_rescue_base_url\": \"https://token.sensenova.cn/v1\",\n \"validation_rescue_api_key_file\": \"config/sensenova_apikeys\",\n \"validation_rescue_attempts\": 2,\n \"validation_quarantine_on_failure\": true,\n \"batch_size\": 16,\n \"num_ctx\": 4096,\n \"temperature\": 0.1,\n \"top_p\": 0.3,\n \"repeat_penalty\": 1.0,\n \"frequency_penalty\": 0.1\n },\n \"galtransl\": {\n \"model\": \"crosery/GalTransl-7B-v2.6:Q6_k\",\n \"host\": \"172.31.102.189\",\n \"timeout_s\": 300,\n \"max_retries\": 4,\n \"batch_size\": 10,\n \"num_ctx\": 4096,\n \"temperature\": 0.2,\n \"top_p\": 0.8,\n \"repeat_penalty\": 1.1,\n \"frequency_penalty\": 0.1\n }\n}\n"}}, "config/quality_final_adjudication.sensenova.json": {"4d507921cd926e659e5a5a22d58eec34686302b8": {"t": 1785768257.9338837, "n": 529, "h": "d6107fc4d7253030", "c": "{\n \"provider\": \"openai\",\n \"base_url\": \"https://token.sensenova.cn/v1\",\n \"api_key_file\": \"config/sensenova_apikeys\",\n \"review_models\": [\n \"sensenova-6.7-flash-lite\",\n \"deepseek-v4-flash\"\n ],\n \"review_fallback_models\": {\n \"sensenova-6.7-flash-lite\": \"glm-5.2\",\n \"deepseek-v4-flash\": \"glm-5.2\"\n },\n \"adjudicator_model\": \"sensenova-6.7-flash-lite\",\n \"adjudicator_fallback_model\": \"deepseek-v4-flash\",\n \"max_workers\": 6,\n \"context_window\": 5,\n \"retries\": 3,\n \"timeout_s\": 600,\n \"min_apply_confidence\": 0.9\n}\n"}}, "scripts/review_agents.py": {"4d507921cd926e659e5a5a22d58eec34686302b8": {"t": 1785768271.7009218, "n": 35793, "h": "23793332ccfffab1", "c": "\"\"\"S11.2: auditable multi-agent subtitle review.\n\nFive reviewer prompts inspect one translated segment in parallel. A separate\neditor may replace the translation only when multiple reviewers agree and the\neditor returns a high-confidence, strictly structured decision. The module is\nsafe to resume on a season-sized job and never mutates the source file.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport hashlib\nimport itertools\nimport json\nimport os\nimport re\nimport threading\nimport time\nimport urllib.error\nimport urllib.request\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import asdict, dataclass, field\nfrom pathlib import Path\nfrom typing import Callable, Optional\n\n\nDEFAULT_MODEL = \"EasonONLINE/Sakura-qwen2.5-v1.0:7b\"\nDEFAULT_SENSENOVA_URL = \"https://token.sensenova.cn/v1\"\nVALID_VERDICTS = {\"ok\", \"fix\", \"suspicious\"}\nVALID_EDITOR_DECISIONS = {\"keep\", \"replace\"}\nMAX_INCOMPLETE_PASSES = 3\nAGENT_SCHEMA = {\n \"type\": \"object\",\n \"properties\": {\n \"verdict\": {\"type\": \"string\", \"enum\": sorted(VALID_VERDICTS)},\n \"suggested_zh\": {\"type\": \"string\"},\n \"reason\": {\"type\": \"string\"},\n \"confidence\": {\"type\": \"number\", \"minimum\": 0, \"maximum\": 1},\n },\n \"required\": [\"verdict\", \"suggested_zh\", \"reason\", \"confidence\"],\n \"additionalProperties\": False,\n}\nEDITOR_SCHEMA = {\n \"type\": \"object\",\n \"properties\": {\n \"decision\": {\"type\": \"string\", \"enum\": sorted(VALID_EDITOR_DECISIONS)},\n \"corrected_zh\": {\"type\": \"string\"},\n \"reason\": {\"type\": \"string\"},\n \"confidence\": {\"type\": \"number\", \"minimum\": 0, \"maximum\": 1},\n },\n \"required\": [\"decision\", \"corrected_zh\", \"reason\", \"confidence\"],\n \"additionalProperties\": False,\n}\n\n\nAGENTS = {\n \"consistency\": {\n \"name\": \"Consistency Checker\",\n \"focus\": \"角色名、称呼、术语和前后台词是否一致\",\n },\n \"naturalness\": {\n \"name\": \"Naturalness Checker\",\n \"focus\": \"中文是否自然、简洁、符合口语和字幕阅读习惯\",\n },\n \"accuracy\": {\n \"name\": \"Accuracy Checker\",\n \"focus\": \"中文是否忠实表达日文含义、语气、主客体,是否漏译或增译\",\n },\n \"asr_check\": {\n \"name\": \"ASR Quality Checker\",\n \"focus\": \"日文 ASR 是否存在误听、断裂、重复或与上下文矛盾\",\n },\n \"style\": {\n \"name\": \"Anime Style Checker\",\n \"focus\": \"译文是否符合轻松动漫对白的语气、人物关系和场景情绪\",\n },\n}\n\n\n@dataclass\nclass ReviewConfig:\n provider: str = field(\n default_factory=lambda: os.environ.get(\"REVIEW_PROVIDER\", \"ollama\")\n )\n host: str = field(\n default_factory=lambda: os.environ.get(\n \"REVIEW_HOST\", os.environ.get(\"OLLAMA_HOST\", \"localhost\")\n )\n )\n review_model: str = field(\n default_factory=lambda: os.environ.get(\"REVIEW_MODEL\", DEFAULT_MODEL)\n )\n review_fallback_model: str = field(\n default_factory=lambda: os.environ.get(\"REVIEW_FALLBACK_MODEL\", \"\")\n )\n base_url: str = field(\n default_factory=lambda: os.environ.get(\"REVIEW_BASE_URL\", \"\")\n )\n api_key_file: str = field(\n default_factory=lambda: os.environ.get(\"REVIEW_API_KEY_FILE\", \"\")\n )\n editor_provider: str = field(\n default_factory=lambda: os.environ.get(\"EDITOR_PROVIDER\", \"\")\n )\n editor_host: str = field(\n default_factory=lambda: os.environ.get(\n \"EDITOR_HOST\",\n os.environ.get(\"REVIEW_HOST\", os.environ.get(\"OLLAMA_HOST\", \"localhost\")),\n )\n )\n editor_model: str = field(\n default_factory=lambda: os.environ.get(\n \"EDITOR_MODEL\", os.environ.get(\"REVIEW_MODEL\", DEFAULT_MODEL)\n )\n )\n editor_fallback_model: str = field(\n default_factory=lambda: os.environ.get(\"EDITOR_FALLBACK_MODEL\", \"\")\n )\n editor_base_url: str = field(\n default_factory=lambda: os.environ.get(\"EDITOR_BASE_URL\", \"\")\n )\n editor_api_key_file: str = field(\n default_factory=lambda: os.environ.get(\"EDITOR_API_KEY_FILE\", \"\")\n )\n agent_models: dict[str, str] = field(default_factory=dict)\n max_workers: int = 2\n context_window: int = 3\n min_fix_votes: int = 2\n min_reviewer_confidence: float = 0.75\n min_editor_confidence: float = 0.80\n retries: int = 3\n timeout_s: int = 300\n\n @classmethod\n def from_dict(cls, data: Optional[dict]) -> \"ReviewConfig\":\n if not data:\n return cls()\n known = {item.name for item in cls.__dataclass_fields__.values()}\n return cls(**{key: value for key, value in data.items() if key in known})\n\n def validate(self) -> \"ReviewConfig\":\n self.provider = self.provider.strip().lower()\n self.review_fallback_model = self.review_fallback_model.strip()\n self.editor_fallback_model = self.editor_fallback_model.strip()\n if self.provider not in {\"ollama\", \"openai\"}:\n raise ValueError(\"provider must be ollama or openai\")\n self.editor_provider = (self.editor_provider or self.provider).strip().lower()\n if self.editor_provider not in {\"ollama\", \"openai\"}:\n raise ValueError(\"editor_provider must be ollama or openai\")\n if self.provider == \"openai\" and not (self.api_key_file or os.environ.get(\"REVIEW_API_KEY\")):\n raise ValueError(\"openai review provider requires api_key_file or REVIEW_API_KEY\")\n if self.editor_provider == \"openai\":\n self.editor_base_url = self.editor_base_url or self.base_url\n self.editor_api_key_file = self.editor_api_key_file or self.api_key_file\n if self.max_workers < 1:\n raise ValueError(\"max_workers must be >= 1\")\n if self.context_window < 0:\n raise ValueError(\"context_window must be >= 0\")\n if not 1 <= self.min_fix_votes <= len(AGENTS):\n raise ValueError(f\"min_fix_votes must be between 1 and {len(AGENTS)}\")\n if not 0 <= self.min_reviewer_confidence <= 1:\n raise ValueError(\"min_reviewer_confidence must be between 0 and 1\")\n if not 0 <= self.min_editor_confidence <= 1:\n raise ValueError(\"min_editor_confidence must be between 0 and 1\")\n if self.retries < 1:\n raise ValueError(\"retries must be >= 1\")\n return self\n\n\ndef _api_url(host: str) -> str:\n host = host.strip().rstrip(\"/\")\n if not host.startswith((\"http://\", \"https://\")):\n host = f\"http://{host}\"\n authority = host.split(\"//\", 1)[1].split(\"/\", 1)[0]\n if \":\" not in authority:\n host = f\"{host}:11434\"\n if host.endswith(\"/api\"):\n return host\n return f\"{host}/api\"\n\n\ndef ollama_chat(\n messages: list[dict],\n *,\n temperature: float = 0.1,\n model: str = DEFAULT_MODEL,\n host: str = \"localhost\",\n json_mode: bool = True,\n json_schema: Optional[dict] = None,\n timeout_s: int = 300,\n) -> str:\n payload = {\n \"model\": model,\n \"messages\": messages,\n \"stream\": False,\n \"options\": {\"temperature\": temperature, \"num_predict\": 768},\n }\n if json_mode:\n payload[\"format\"] = json_schema or \"json\"\n request = urllib.request.Request(\n f\"{_api_url(host)}/chat\",\n data=json.dumps(payload, ensure_ascii=False).encode(\"utf-8\"),\n headers={\"Content-Type\": \"application/json; charset=utf-8\"},\n )\n with urllib.request.urlopen(request, timeout=timeout_s) as response:\n result = json.loads(response.read().decode(\"utf-8\"))\n return result.get(\"message\", {}).get(\"content\", \"\").strip()\n\n\n_KEY_SEQUENCE = itertools.count()\n_KEY_LOCK = threading.Lock()\n\n\ndef _api_keys(path: str, environment_name: str = \"REVIEW_API_KEY\") -> list[str]:\n keys = []\n environment_value = os.environ.get(environment_name, \"\").strip()\n if environment_value:\n keys.append(environment_value)\n if path:\n keys.extend(\n line.strip()\n for line in Path(path).read_text(encoding=\"utf-8\").splitlines()\n if line.strip()\n )\n # Preserve order while removing accidental duplicates.\n return list(dict.fromkeys(keys))\n\n\ndef _next_api_key(path: str) -> str:\n keys = _api_keys(path)\n if not keys:\n raise ValueError(\"no API key is available\")\n with _KEY_LOCK:\n position = next(_KEY_SEQUENCE)\n return keys[position % len(keys)]\n\n\ndef openai_chat(\n messages: list[dict],\n *,\n temperature: float = 0.1,\n model: str,\n base_url: str,\n api_key_file: str,\n json_mode: bool = True,\n timeout_s: int = 300,\n **_kwargs,\n) -> str:\n payload = {\n \"model\": model,\n \"messages\": messages,\n \"stream\": False,\n \"temperature\": temperature,\n \"max_tokens\": 2048,\n }\n if json_mode:\n # SenseNova's OpenAI-compatible gateway supports json_object. The\n # semantic schema remains in the prompt and is validated locally.\n payload[\"response_format\"] = {\"type\": \"json_object\"}\n endpoint = (base_url or DEFAULT_SENSENOVA_URL).rstrip(\"/\") + \"/chat/completions\"\n request = urllib.request.Request(\n endpoint,\n data=json.dumps(payload, ensure_ascii=False).encode(\"utf-8\"),\n headers={\n \"Authorization\": f\"Bearer {_next_api_key(api_key_file)}\",\n \"Content-Type\": \"application/json; charset=utf-8\",\n \"Accept\": \"application/json\",\n },\n )\n try:\n with urllib.request.urlopen(request, timeout=timeout_s) as response:\n result = json.loads(response.read().decode(\"utf-8\"))\n except urllib.error.HTTPError as error:\n # Do not include response bodies: providers sometimes echo request\n # metadata, and review logs must never risk persisting credentials.\n raise RuntimeError(f\"OpenAI-compatible API returned HTTP {error.code}\") from error\n choices = result.get(\"choices\") or []\n if not choices:\n raise ValueError(\"OpenAI-compatible API response has no choices\")\n return str(choices[0].get(\"message\", {}).get(\"content\", \"\")).strip()\n\n\ndef model_chat(\n messages: list[dict],\n *,\n provider: str = \"ollama\",\n base_url: str = \"\",\n api_key_file: str = \"\",\n **kwargs,\n) -> str:\n if provider == \"openai\":\n return openai_chat(\n messages,\n base_url=base_url,\n api_key_file=api_key_file,\n **kwargs,\n )\n return ollama_chat(messages, **kwargs)\n\n\ndef _json_object(response: str) -> tuple[dict, bool]:\n \"\"\"Parse one JSON object with only deterministic syntax repairs.\n\n Sakura occasionally stops after a complete final value but before the\n closing brace. Appending that single brace (or removing a trailing comma)\n cannot invent a verdict; all semantic fields are still schema-validated by\n the caller. Natural-language keyword guessing remains deliberately banned.\n \"\"\"\n text = (response or \"\").strip()\n fenced = re.fullmatch(r\"```(?:json)?\\s*(\\{.*\\})\\s*```\", text, re.DOTALL)\n if fenced:\n text = fenced.group(1)\n repaired = False\n try:\n value = json.loads(text)\n except json.JSONDecodeError as original_error:\n candidates = []\n without_trailing_comma = re.sub(r\",\\s*}$\", \"}\", text)\n if without_trailing_comma != text:\n candidates.append(without_trailing_comma)\n if (\n text.startswith(\"{\")\n and text.count(\"{\") == text.count(\"}\") + 1\n and len(re.findall(r'(?<!\\\\)\"', text)) % 2 == 0\n ):\n candidates.append(text + \"}\")\n value = None\n for candidate in candidates:\n try:\n value = json.loads(candidate)\n repaired = True\n break\n except json.JSONDecodeError:\n continue\n if value is None:\n raise ValueError(\n f\"response is not a JSON object: {original_error.msg}\"\n ) from original_error\n if not isinstance(value, dict):\n raise ValueError(\"response JSON must be an object\")\n return value, repaired\n\n\ndef _strict_object(value: dict, required: set[str], label: str) -> None:\n keys = set(value)\n missing = required - keys\n extra = keys - required\n if missing:\n raise ValueError(f\"{label} response is missing fields: {sorted(missing)}\")\n if extra:\n raise ValueError(f\"{label} response has unexpected fields: {sorted(extra)}\")\n\n\ndef _strict_confidence(value, label: str) -> float:\n if isinstance(value, bool) or not isinstance(value, (int, float)):\n raise ValueError(f\"{label} confidence must be a JSON number\")\n confidence = float(value)\n if not 0 <= confidence <= 1:\n raise ValueError(f\"{label} confidence must be between 0 and 1\")\n return confidence\n\n\ndef parse_agent_response(response: str) -> dict:\n value, repaired = _json_object(response)\n _strict_object(\n value, {\"verdict\", \"suggested_zh\", \"reason\", \"confidence\"}, \"reviewer\"\n )\n if not all(isinstance(value[key], str) for key in (\"verdict\", \"suggested_zh\", \"reason\")):\n raise ValueError(\"reviewer verdict, suggested_zh and reason must be strings\")\n verdict = str(value.get(\"verdict\", \"\")).strip().lower()\n if verdict not in VALID_VERDICTS:\n raise ValueError(f\"invalid reviewer verdict: {verdict!r}\")\n suggested = str(value.get(\"suggested_zh\") or \"\").strip()\n if verdict == \"fix\" and not suggested:\n raise ValueError(\"fix verdict requires suggested_zh\")\n return {\n \"verdict\": verdict,\n \"suggested_zh\": suggested or None,\n \"reason\": str(value.get(\"reason\") or \"\").strip(),\n \"confidence\": _strict_confidence(value[\"confidence\"], \"reviewer\"),\n \"response_repaired\": repaired,\n }\n\n\ndef parse_editor_response(response: str) -> dict:\n value, repaired = _json_object(response)\n _strict_object(\n value, {\"decision\", \"corrected_zh\", \"reason\", \"confidence\"}, \"editor\"\n )\n if not all(isinstance(value[key], str) for key in (\"decision\", \"corrected_zh\", \"reason\")):\n raise ValueError(\"editor decision, corrected_zh and reason must be strings\")\n decision = str(value.get(\"decision\", \"\")).strip().lower()\n if decision not in VALID_EDITOR_DECISIONS:\n raise ValueError(f\"invalid editor decision: {decision!r}\")\n corrected = str(value.get(\"corrected_zh\") or \"\").strip()\n if decision == \"replace\" and not corrected:\n raise ValueError(\"replace decision requires corrected_zh\")\n return {\n \"decision\": decision,\n \"corrected_zh\": corrected or None,\n \"reason\": str(value.get(\"reason\") or \"\").strip(),\n \"confidence\": _strict_confidence(value[\"confidence\"], \"editor\"),\n \"response_repaired\": repaired,\n }\n\n\ndef _segment_text(segment: dict) -> str:\n return str(segment.get(\"text\", segment.get(\"zh\", \"\"))).strip()\n\n\ndef _segment_key(position: int, segment: dict) -> str:\n payload = {\n \"position\": position,\n \"id\": segment.get(\"id\", segment.get(\"index\")),\n \"ja\": str(segment.get(\"ja\", \"\")),\n \"text\": _segment_text(segment),\n }\n digest = hashlib.sha256(\n json.dumps(payload, ensure_ascii=False, sort_keys=True).encode(\"utf-8\")\n ).hexdigest()[:20]\n return f\"{position}:{digest}\"\n\n\ndef _context_for(segments: list[dict], position: int, window: int) -> str:\n start = max(0, position - window)\n end = min(len(segments), position + window + 1)\n lines = []\n for index in range(start, end):\n marker = \"目标\" if index == position else \"上下文\"\n segment = segments[index]\n lines.append(\n f\"[{marker} {index}] 日文:{segment.get('ja', '')}\\n\"\n f\"[{marker} {index}] 中文:{_segment_text(segment)}\"\n )\n return \"\\n\".join(lines)\n\n\ndef load_glossary_prompt(path: str = \"\") -> str:\n if not path:\n return \"(未提供术语表)\"\n value = json.loads(Path(path).read_text(encoding=\"utf-8\"))\n terms = value.get(\"terms\", value) if isinstance(value, dict) else value\n rows = []\n if isinstance(terms, list):\n for item in terms:\n if isinstance(item, dict) and item.get(\"ja\") and item.get(\"zh\"):\n rows.append(f\"{item['ja']} => {item['zh']}\")\n elif isinstance(terms, dict):\n rows.extend(f\"{ja} => {zh}\" for ja, zh in terms.items())\n return \"\\n\".join(rows) if rows else \"(术语表为空)\"\n\n\ndef _call_json_with_retries(\n messages: list[dict],\n *,\n model: str,\n host: str,\n provider: str,\n base_url: str,\n api_key_file: str,\n config: ReviewConfig,\n parser: Callable[[str], dict],\n json_schema: dict,\n chat_fn: Callable = model_chat,\n temperature: float = 0.1,\n) -> tuple[Optional[dict], str, str, int, float]:\n last_response = \"\"\n last_error = \"\"\n started = time.time()\n for attempt in range(1, config.retries + 1):\n try:\n last_response = chat_fn(\n messages,\n temperature=temperature,\n model=model,\n host=host,\n provider=provider,\n base_url=base_url,\n api_key_file=api_key_file,\n json_mode=True,\n json_schema=json_schema,\n timeout_s=config.timeout_s,\n )\n return parser(last_response), last_response, \"\", attempt, time.time() - started\n except Exception as error: # network and model-format failures are retryable\n last_error = str(error)\n if attempt < config.retries:\n time.sleep(min(2 ** (attempt - 1), 4))\n return None, last_response, last_error, config.retries, time.time() - started\n\n\ndef _call_json_with_fallback(\n messages: list[dict],\n *,\n model: str,\n fallback_model: str,\n host: str,\n provider: str,\n base_url: str,\n api_key_file: str,\n config: ReviewConfig,\n parser: Callable[[str], dict],\n json_schema: dict,\n chat_fn: Callable = model_chat,\n temperature: float = 0.1,\n) -> tuple[Optional[dict], str, str, int, float, str, bool, str]:\n primary = _call_json_with_retries(\n messages,\n model=model,\n host=host,\n provider=provider,\n base_url=base_url,\n api_key_file=api_key_file,\n config=config,\n parser=parser,\n json_schema=json_schema,\n chat_fn=chat_fn,\n temperature=temperature,\n )\n parsed, raw, error, attempts, elapsed = primary\n fallback_model = fallback_model.strip()\n if parsed is not None or not fallback_model or fallback_model == model:\n return parsed, raw, error, attempts, elapsed, model, False, \"\"\n\n primary_error = error\n fallback = _call_json_with_retries(\n messages,\n model=fallback_model,\n host=host,\n provider=provider,\n base_url=base_url,\n api_key_file=api_key_file,\n config=config,\n parser=parser,\n json_schema=json_schema,\n chat_fn=chat_fn,\n temperature=temperature,\n )\n parsed, raw, fallback_error, fallback_attempts, fallback_elapsed = fallback\n if parsed is None:\n error = (\n f\"primary {model}: {primary_error}; \"\n f\"fallback {fallback_model}: {fallback_error}\"\n )\n else:\n error = \"\"\n return (\n parsed,\n raw,\n error,\n attempts + fallback_attempts,\n elapsed + fallback_elapsed,\n fallback_model,\n True,\n primary_error,\n )\n\n\ndef run_agent(\n agent_id: str,\n segment: dict,\n context: str,\n glossary: str,\n config: ReviewConfig,\n chat_fn: Callable = model_chat,\n) -> dict:\n agent = AGENTS[agent_id]\n ja = str(segment.get(\"ja\", \"\")).strip()\n zh = _segment_text(segment)\n system = (\n f\"你是{agent['name']},只负责检查:{agent['focus']}。\"\n \"不要为了追求不同而改写正确译文。只能输出一个 JSON 对象,不要复述提示词。\"\n )\n prompt = f\"\"\"请审查目标字幕。\n\n上下文:\n{context}\n\n目标日文:{ja}\n目标中文:{zh}\n\n必须遵守的术语:\n{glossary}\n\n只输出:\n{{\"verdict\":\"ok|fix|suspicious\",\"suggested_zh\":\"完整建议译文,ok 时为空字符串\",\"reason\":\"简短依据\",\"confidence\":0.0}}\nconfidence 范围为 0 到 1。ASR 可疑用 suspicious;翻译确有错误才用 fix。\"\"\"\n model = config.agent_models.get(agent_id, config.review_model)\n (\n parsed,\n raw,\n error,\n attempts,\n elapsed,\n used_model,\n fallback_used,\n primary_error,\n ) = _call_json_with_fallback(\n [{\"role\": \"system\", \"content\": system}, {\"role\": \"user\", \"content\": prompt}],\n model=model,\n fallback_model=config.review_fallback_model,\n host=config.host,\n provider=config.provider,\n base_url=config.base_url,\n api_key_file=config.api_key_file,\n config=config,\n parser=parse_agent_response,\n json_schema=AGENT_SCHEMA,\n chat_fn=chat_fn,\n )\n result = {\n \"agent\": agent_id,\n \"agent_name\": agent[\"name\"],\n \"model\": used_model,\n \"primary_model\": model,\n \"fallback_used\": fallback_used,\n \"primary_error\": primary_error or None,\n \"attempts\": attempts,\n \"time_s\": round(elapsed, 2),\n \"raw_response\": raw,\n }\n if parsed is None:\n result.update(\n verdict=\"error\", suggested_zh=None, reason=error,\n confidence=0.0, error=error,\n )\n else:\n result.update(parsed)\n return result\n\n\ndef run_editor(\n segment: dict,\n context: str,\n glossary: str,\n agent_results: dict[str, dict],\n config: ReviewConfig,\n chat_fn: Callable = model_chat,\n) -> dict:\n feedback = [\n {\n \"agent\": agent_id,\n \"verdict\": result.get(\"verdict\"),\n \"suggested_zh\": result.get(\"suggested_zh\"),\n \"reason\": result.get(\"reason\"),\n \"confidence\": result.get(\"confidence\"),\n }\n for agent_id in AGENTS\n for result in [agent_results[agent_id]]\n ]\n system = (\n \"你是保守的字幕总编。只有审查意见揭示真实语义、术语、ASR 或中文表达问题时才替换;\"\n \"意见冲突、证据不足或只是同义改写时必须保留原译。只能输出一个 JSON 对象。\"\n )\n prompt = f\"\"\"上下文:\n{context}\n\n目标日文:{segment.get('ja', '')}\n原中文:{_segment_text(segment)}\n\n术语:\n{glossary}\n\n审查意见:\n{json.dumps(feedback, ensure_ascii=False)}\n\n只输出:\n{{\"decision\":\"keep|replace\",\"corrected_zh\":\"完整最终译文,keep 时可为空\",\"reason\":\"依据\",\"confidence\":0.0}}\"\"\"\n (\n parsed,\n raw,\n error,\n attempts,\n elapsed,\n used_model,\n fallback_used,\n primary_error,\n ) = _call_json_with_fallback(\n [{\"role\": \"system\", \"content\": system}, {\"role\": \"user\", \"content\": prompt}],\n model=config.editor_model,\n fallback_model=config.editor_fallback_model,\n host=config.editor_host,\n provider=config.editor_provider,\n base_url=config.editor_base_url,\n api_key_file=config.editor_api_key_file,\n config=config,\n parser=parse_editor_response,\n json_schema=EDITOR_SCHEMA,\n chat_fn=chat_fn,\n temperature=0.05,\n )\n result = {\n \"model\": used_model,\n \"primary_model\": config.editor_model,\n \"fallback_used\": fallback_used,\n \"primary_error\": primary_error or None,\n \"attempts\": attempts,\n \"time_s\": round(elapsed, 2),\n \"raw_response\": raw,\n }\n if parsed is None:\n result.update(\n decision=\"error\", corrected_zh=None, reason=error,\n confidence=0.0, error=error,\n )\n else:\n result.update(parsed)\n return result\n\n\ndef review_segment(\n position: int,\n segment: dict,\n context: str,\n glossary: str,\n config: ReviewConfig,\n chat_fn: Callable = model_chat,\n) -> dict:\n results = {}\n with ThreadPoolExecutor(max_workers=len(AGENTS)) as executor:\n futures = {\n executor.submit(\n run_agent, agent_id, segment, context, glossary, config, chat_fn\n ): agent_id\n for agent_id in AGENTS\n }\n for future in as_completed(futures):\n agent_id = futures[future]\n try:\n results[agent_id] = future.result()\n except Exception as error:\n results[agent_id] = {\n \"agent\": agent_id,\n \"agent_name\": AGENTS[agent_id][\"name\"],\n \"verdict\": \"error\",\n \"reason\": str(error),\n \"confidence\": 0.0,\n \"suggested_zh\": None,\n }\n\n results = {agent_id: results[agent_id] for agent_id in AGENTS}\n fix_votes = sum(\n result.get(\"verdict\") in {\"fix\", \"suspicious\"}\n and result.get(\"confidence\", 0.0) >= config.min_reviewer_confidence\n for result in results.values()\n )\n editor = None\n if fix_votes >= config.min_fix_votes:\n editor = run_editor(segment, context, glossary, results, config, chat_fn)\n\n original = _segment_text(segment)\n errors = sum(result.get(\"verdict\") == \"error\" for result in results.values())\n editor_error = bool(editor and editor.get(\"decision\") == \"error\")\n applied = bool(\n editor\n and errors == 0\n and not editor_error\n and editor.get(\"decision\") == \"replace\"\n and editor.get(\"corrected_zh\")\n and editor.get(\"corrected_zh\") != original\n and editor.get(\"confidence\", 0.0) >= config.min_editor_confidence\n )\n status = \"corrected\" if applied else \"approved\"\n if errors or editor_error:\n status = \"error\"\n elif fix_votes and not applied:\n status = \"needs_review\"\n\n return {\n \"position\": position,\n \"segment_key\": _segment_key(position, segment),\n \"ja\": str(segment.get(\"ja\", \"\")),\n \"original_zh\": original,\n \"final_zh\": editor[\"corrected_zh\"] if applied else original,\n \"status\": status,\n \"fix_votes\": fix_votes,\n \"applied\": applied,\n \"agent_results\": results,\n \"editor_result\": editor,\n }\n\n\ndef _load_progress(path: Optional[Path]) -> dict[str, dict]:\n completed = {}\n if not path or not path.exists():\n return completed\n with path.open(encoding=\"utf-8\") as file:\n for line in file:\n try:\n item = json.loads(line)\n except json.JSONDecodeError:\n continue\n key = item.get(\"segment_key\")\n if not key:\n continue\n editor_result = item.get(\"editor_result\") or {}\n if (\n item.get(\"status\") == \"error\"\n or editor_result.get(\"decision\") == \"error\"\n ):\n completed.pop(key, None)\n else:\n completed[key] = item\n return completed\n\n\ndef _append_progress(path: Optional[Path], item: dict) -> None:\n if not path:\n return\n path.parent.mkdir(parents=True, exist_ok=True)\n with path.open(\"a\", encoding=\"utf-8\") as file:\n file.write(json.dumps(item, ensure_ascii=False) + \"\\n\")\n file.flush()\n\n\ndef review_batch(\n segments: list[dict],\n max_workers: Optional[int] = None,\n *,\n config: Optional[ReviewConfig] = None,\n glossary: str = \"(未提供术语表)\",\n progress_path: str = \"\",\n completed_results: Optional[dict[str, dict]] = None,\n chat_fn: Callable = model_chat,\n) -> list[dict]:\n config = config or ReviewConfig()\n if max_workers is not None:\n config.max_workers = max_workers\n config.validate()\n progress = Path(progress_path) if progress_path else None\n cached = _load_progress(progress)\n cached.update(completed_results or {})\n results: list[Optional[dict]] = [None] * len(segments)\n pending = []\n for position, segment in enumerate(segments):\n key = _segment_key(position, segment)\n if key in cached:\n results[position] = cached[key]\n else:\n pending.append(position)\n\n print(\n f\"Reviewing {len(segments)} segments with {len(AGENTS)} agents; \"\n f\"resume={len(segments) - len(pending)}, pending={len(pending)}\"\n )\n print(f\"Reviewer model: {config.review_model}; editor: {config.editor_model}\")\n\n with ThreadPoolExecutor(max_workers=config.max_workers) as executor:\n futures = {\n executor.submit(\n review_segment,\n position,\n segments[position],\n _context_for(segments, position, config.context_window),\n glossary,\n config,\n chat_fn,\n ): position\n for position in pending\n }\n for future in as_completed(futures):\n position = futures[future]\n item = future.result()\n results[position] = item\n _append_progress(progress, item)\n print(\n f\" [{position + 1}/{len(segments)}] {item['status']} \"\n f\"votes={item['fix_votes']}\"\n )\n\n return [item for item in results if item is not None]\n\n\ndef _atomic_json(path: Path, value) -> None:\n path.parent.mkdir(parents=True, exist_ok=True)\n temporary = path.with_suffix(path.suffix + \".tmp\")\n temporary.write_text(\n json.dumps(value, ensure_ascii=False, indent=2), encoding=\"utf-8\"\n )\n temporary.replace(path)\n\n\ndef review_translation_file(\n input_path: str,\n reviewed_path: str,\n report_path: str,\n *,\n progress_path: str = \"\",\n glossary_path: str = \"\",\n config: Optional[ReviewConfig] = None,\n apply_fixes: bool = True,\n chat_fn: Callable = model_chat,\n) -> dict:\n source = Path(input_path)\n segments = json.loads(source.read_text(encoding=\"utf-8\"))\n if not isinstance(segments, list):\n raise ValueError(\"translated input must be a JSON list\")\n config = (config or ReviewConfig()).validate()\n glossary = load_glossary_prompt(glossary_path)\n reviews = []\n completed_results = {}\n for pass_index in range(1, MAX_INCOMPLETE_PASSES + 1):\n reviews = review_batch(\n segments,\n config=config,\n glossary=glossary,\n progress_path=progress_path,\n completed_results=completed_results,\n chat_fn=chat_fn,\n )\n error_count = sum(item[\"status\"] == \"error\" for item in reviews)\n completed_results.update(\n (item[\"segment_key\"], item)\n for item in reviews\n if item[\"status\"] != \"error\"\n )\n if error_count == 0:\n break\n if pass_index < MAX_INCOMPLETE_PASSES:\n print(\n f\"Retrying {error_count} incomplete multi-agent review(s); \"\n f\"pass {pass_index + 1}/{MAX_INCOMPLETE_PASSES}\"\n )\n\n reviewed_segments = []\n for segment, review in zip(segments, reviews):\n output = dict(segment)\n if apply_fixes and review[\"applied\"]:\n output[\"text\"] = review[\"final_zh\"]\n output[\"multi_agent_review\"] = {\n \"segment_key\": review[\"segment_key\"],\n \"status\": review[\"status\"],\n \"fix_votes\": review[\"fix_votes\"],\n \"applied\": bool(apply_fixes and review[\"applied\"]),\n \"original_zh\": review[\"original_zh\"],\n \"final_zh\": output.get(\"text\", _segment_text(output)),\n \"editor\": review.get(\"editor_result\"),\n }\n reviewed_segments.append(output)\n\n summary = {\n \"total\": len(reviews),\n \"approved\": sum(item[\"status\"] == \"approved\" for item in reviews),\n \"corrected\": sum(item[\"status\"] == \"corrected\" for item in reviews),\n \"needs_review\": sum(item[\"status\"] == \"needs_review\" for item in reviews),\n \"errors\": sum(item[\"status\"] == \"error\" for item in reviews),\n \"applied\": sum(bool(apply_fixes and item[\"applied\"]) for item in reviews),\n }\n report = {\n \"schema\": \"anime-accurate-sub/multi-agent-review-v2\",\n \"source\": str(source.resolve()),\n \"config\": asdict(config),\n \"apply_fixes\": apply_fixes,\n \"summary\": summary,\n \"reviews\": reviews,\n }\n _atomic_json(Path(reviewed_path), reviewed_segments)\n _atomic_json(Path(report_path), report)\n if summary[\"errors\"]:\n raise RuntimeError(\n f\"multi-agent review has {summary['errors']} incomplete segment(s); \"\n \"rerun with the same progress path\"\n )\n return report\n\n\ndef create_test_segments() -> list[dict]:\n return [\n {\"ja\": \"おはよう、唯\", \"text\": \"早安,唯\"},\n {\"ja\": \"今日も元気だね\", \"text\": \"今天也很有精神呢\"},\n {\"ja\": \"澪ちゃん、一緒に練習しない?\", \"text\": \"澪,要不要一起练习?\"},\n {\"ja\": \"ありがとう、唯。じゃあ放課後ね\", \"text\": \"谢谢你,唯。那放学后见\"},\n ]\n\n\ndef evaluate() -> None:\n output_root = Path(__file__).resolve().parent.parent / \"docs\" / \"evaluation\"\n source = output_root / \"S11.2_test_segments.json\"\n reviewed = output_root / \"S11.2_reviewed_segments.json\"\n report = output_root / \"S11.2_review_results.json\"\n _atomic_json(source, create_test_segments())\n result = review_translation_file(\n str(source), str(reviewed), str(report), apply_fixes=True\n )\n print(json.dumps(result[\"summary\"], ensure_ascii=False, indent=2))\n\n\ndef _load_config(path: str, args) -> ReviewConfig:\n data = {}\n if path:\n data = json.loads(Path(path).read_text(encoding=\"utf-8\"))\n overrides = {\n \"provider\": args.provider,\n \"host\": args.host,\n \"review_model\": args.model,\n \"base_url\": args.base_url,\n \"api_key_file\": args.api_key_file,\n \"editor_provider\": args.editor_provider,\n \"editor_host\": args.editor_host,\n \"editor_model\": args.editor_model,\n \"editor_base_url\": args.editor_base_url,\n \"editor_api_key_file\": args.editor_api_key_file,\n \"max_workers\": args.workers,\n \"min_fix_votes\": args.min_fix_votes,\n \"min_reviewer_confidence\": args.min_reviewer_confidence,\n \"min_editor_confidence\": args.min_editor_confidence,\n \"context_window\": args.context_window,\n }\n data.update({key: value for key, value in overrides.items() if value is not None})\n return ReviewConfig.from_dict(data).validate()\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(description=\"S11.2 auditable multi-agent review\")\n parser.add_argument(\"--input\", help=\"translated JSON list\")\n parser.add_argument(\"--output\", default=\"reviewed.json\", help=\"reviewed JSON\")\n parser.add_argument(\"--report\", default=\"multi_agent_review.json\")\n parser.add_argument(\"--progress\", default=\"\", help=\"resumable JSONL progress\")\n parser.add_argument(\"--glossary\", default=\"\")\n parser.add_argument(\"--config\", default=\"\", help=\"review configuration JSON\")\n parser.add_argument(\"--host\")\n parser.add_argument(\"--model\")\n parser.add_argument(\"--provider\", choices=(\"ollama\", \"openai\"))\n parser.add_argument(\"--base-url\")\n parser.add_argument(\"--api-key-file\")\n parser.add_argument(\"--editor-host\")\n parser.add_argument(\"--editor-model\")\n parser.add_argument(\"--editor-provider\", choices=(\"ollama\", \"openai\"))\n parser.add_argument(\"--editor-base-url\")\n parser.add_argument(\"--editor-api-key-file\")\n parser.add_argument(\"--workers\", type=int)\n parser.add_argument(\"--min-fix-votes\", type=int)\n parser.add_argument(\"--min-reviewer-confidence\", type=float)\n parser.add_argument(\"--min-editor-confidence\", type=float)\n parser.add_argument(\"--context-window\", type=int)\n parser.add_argument(\n \"--dry-run\", action=\"store_true\", help=\"review but do not apply corrections\"\n )\n parser.add_argument(\"--evaluate\", action=\"store_true\")\n args = parser.parse_args()\n\n if args.evaluate:\n evaluate()\n return\n if not args.input:\n parser.error(\"--input is required unless --evaluate is used\")\n result = review_translation_file(\n args.input,\n args.output,\n args.report,\n progress_path=args.progress,\n glossary_path=args.glossary,\n config=_load_config(args.config, args),\n apply_fixes=not args.dry_run,\n )\n print(json.dumps(result[\"summary\"], ensure_ascii=False, indent=2))\n\n\nif __name__ == \"__main__\":\n main()\n"}}, "scripts/translator_adapter.py": {"4d507921cd926e659e5a5a22d58eec34686302b8": {"t": 1785768275.1200373, "n": 35279, "h": "bf9c45edb7a58a03", "c": "# S14.1: Translator Adapter - abstract base + multi-backend implementations\n#\n# Architecture:\n# TranslatorAdapter (ABC)\n# +-- SakuraAdapter (Ollama Sakura-7B/14B)\n# +-- QwenAdapter (Ollama Qwen2.5)\n# +-- GalTranslAdapter (Ollama GalTransl-7B)\n# +-- ExternalAdapter (OpenAI-compatible API)\n#\n# Config file format:\n# {\n# \"backend\": \"sakura\",\n# \"sakura\": {\"model\": \"crosery/sakura-14b-qwen2.5-v1.0-q6k:latest\", \"host\": \"172.31.102.189\"},\n# \"qwen\": {\"model\": \"qwen2.5:7b\", \"host\": \"localhost\"},\n# \"galtransl\": {\"model\": \"crosery/GalTransl-7B-v2.6:Q6_k\", \"host\": \"172.31.102.189\"},\n# \"external\": {\"api_url\": \"...\", \"api_key\": \"...\", \"model\": \"...\"}\n# }\n#\n# Usage:\n# python scripts/translator_adapter.py --text \"おはよう\" --backend sakura\n# python scripts/translator_adapter.py --text \"おはよう\" --config translator_config.json\n# python scripts/translator_adapter.py --list-backends\n# python scripts/translator_adapter.py --evaluate\n\nimport abc\nimport argparse\nimport inspect\nimport json\nimport re\nimport socket\nimport sys\nimport time\nimport unicodedata\nimport urllib.error\nimport urllib.request\nfrom pathlib import Path\nfrom typing import Sequence\n\nproject_root = Path(__file__).resolve().parent.parent\nsys.path.insert(0, str(project_root))\n\nfrom scripts.plugin_system import load_plugins, plugin_registry # noqa: E402\n\nDEFAULT_CONFIG = {\n \"backend\": \"sakura\",\n \"sakura\": {\n \"model\": \"crosery/sakura-14b-qwen2.5-v1.0-q6k:latest\",\n \"host\": \"172.31.102.189\",\n \"timeout_s\": 300,\n \"max_retries\": 4,\n \"validation_retries\": 2,\n \"validation_fallback_backend\": \"galtransl\",\n \"validation_rescue_models\": [\n \"sensenova-6.7-flash-lite\",\n \"deepseek-v4-flash\",\n ],\n \"validation_rescue_base_url\": \"https://token.sensenova.cn/v1\",\n \"validation_rescue_api_key_file\": \"config/sensenova_apikeys\",\n \"validation_rescue_attempts\": 2,\n \"validation_quarantine_on_failure\": True,\n \"batch_size\": 16,\n \"num_ctx\": 4096,\n \"temperature\": 0.1,\n \"top_p\": 0.3,\n \"repeat_penalty\": 1.0,\n \"frequency_penalty\": 0.1,\n },\n \"qwen\": {\n \"model\": \"qwen2.5:7b\",\n \"host\": \"localhost\",\n },\n \"galtransl\": {\n \"model\": \"crosery/GalTransl-7B-v2.6:Q6_k\",\n \"host\": \"172.31.102.189\",\n \"timeout_s\": 300,\n \"max_retries\": 4,\n \"batch_size\": 10,\n \"num_ctx\": 4096,\n \"temperature\": 0.2,\n \"top_p\": 0.8,\n \"repeat_penalty\": 1.1,\n \"frequency_penalty\": 0.1,\n },\n \"external\": {\n \"api_url\": \"\",\n \"api_key\": \"\",\n \"model\": \"\",\n },\n}\n\n\ndef _call_batch_with_optional_context(\n adapter,\n texts,\n glossary_terms,\n context_before=None,\n context_after=None,\n):\n \"\"\"Call old and new translator plugins without masking adapter errors.\"\"\"\n method = adapter.translate_batch\n try:\n parameters = inspect.signature(method).parameters.values()\n supports_keywords = any(\n parameter.kind == inspect.Parameter.VAR_KEYWORD\n for parameter in parameters\n )\n parameter_names = {parameter.name for parameter in parameters}\n except (TypeError, ValueError):\n supports_keywords = False\n parameter_names = set()\n if supports_keywords or {\"context_before\", \"context_after\"} <= parameter_names:\n return method(\n texts,\n glossary_terms,\n context_before=context_before,\n context_after=context_after,\n )\n return method(texts, glossary_terms)\n\n\n# ============ Abstract Base ============\n\nclass TranslatorAdapter(abc.ABC):\n \"\"\"Abstract base class for all translation backends.\"\"\"\n\n def __init__(self, config: dict):\n self.config = config\n self.series_info = \"\"\n self._translation_models: dict[str, str] = {}\n self._translation_fallbacks: dict[str, bool] = {}\n self._translation_errors: dict[str, str] = {}\n\n @abc.abstractmethod\n def translate(self, text: str, context_before=None, context_after=None) -> str:\n \"\"\"Translate Japanese text to Chinese.\"\"\"\n pass\n\n def translate_batch(\n self,\n texts: Sequence[str],\n glossary_terms=None,\n context_before=None,\n context_after=None,\n ) -> list[str]:\n \"\"\"Translate ordered lines, falling back to the single-line interface.\"\"\"\n texts = list(texts)\n outer_before = list(context_before or [])\n outer_after = list(context_after or [])\n return [\n self.translate(\n text,\n context_before=outer_before + texts[:index],\n context_after=texts[index + 1:] + outer_after,\n )\n for index, text in enumerate(texts)\n ]\n\n def result_model(self, source: str) -> str:\n \"\"\"Return the model that produced a source line's latest translation.\"\"\"\n return self._translation_models.get(\n source,\n getattr(self, \"model\", self.name()),\n )\n\n def result_is_fallback(self, source: str) -> bool:\n return self._translation_fallbacks.get(source, False)\n\n def result_error(self, source: str) -> str | None:\n \"\"\"Return a terminal recovery error when a line was quarantined.\"\"\"\n return self._translation_errors.get(source)\n\n @abc.abstractmethod\n def name(self) -> str:\n \"\"\"Return backend name for display.\"\"\"\n pass\n\n @classmethod\n def from_config(cls, config: dict) -> \"TranslatorAdapter\":\n \"\"\"Factory: create adapter from config.\"\"\"\n _ensure_builtin_translator_plugins()\n backend = str(config.get(\"backend\", \"sakura\")).lower()\n return plugin_registry.create(\"translator\", backend, config)\n\n\n# ============ Ollama Base ============\n\nclass OllamaAdapter(TranslatorAdapter):\n \"\"\"Base for Ollama-based adapters.\"\"\"\n\n def __init__(self, config: dict, backend_key: str):\n super().__init__(config)\n backend_cfg = config.get(backend_key, {})\n self.model = backend_cfg.get(\"model\", \"\")\n self.host = str(backend_cfg.get(\"host\", \"localhost\")).rstrip(\"/\")\n if \"://\" in self.host:\n base_url = self.host\n elif re.search(r\":\\d+$\", self.host):\n base_url = f\"http://{self.host}\"\n else:\n base_url = f\"http://{self.host}:11434\"\n self.api_url = f\"{base_url}/api/chat\"\n self.timeout_s = int(backend_cfg.get(\"timeout_s\", 300))\n self.max_retries = int(backend_cfg.get(\"max_retries\", 4))\n self.retry_backoff_s = float(backend_cfg.get(\"retry_backoff_s\", 2.0))\n self.batch_size = int(backend_cfg.get(\"batch_size\", 16))\n self.num_ctx = int(backend_cfg.get(\"num_ctx\", 4096))\n self.temperature = float(backend_cfg.get(\"temperature\", 0.1))\n self.top_p = float(backend_cfg.get(\"top_p\", 0.3))\n self.repeat_penalty = float(backend_cfg.get(\"repeat_penalty\", 1.0))\n self.frequency_penalty = float(backend_cfg.get(\"frequency_penalty\", 0.1))\n self.validation_retries = int(backend_cfg.get(\"validation_retries\", 2))\n\n def _call(self, messages, num_predict=1024):\n payload = {\n \"model\": self.model,\n \"messages\": messages,\n \"stream\": False,\n \"options\": {\n \"temperature\": self.temperature,\n \"top_p\": self.top_p,\n \"repeat_penalty\": self.repeat_penalty,\n \"num_predict\": num_predict,\n \"frequency_penalty\": self.frequency_penalty,\n \"num_ctx\": self.num_ctx,\n },\n \"keep_alive\": \"30m\",\n }\n data = json.dumps(payload, ensure_ascii=False).encode(\"utf-8\")\n last_error = None\n for attempt in range(1, self.max_retries + 1):\n req = urllib.request.Request(\n self.api_url,\n data=data,\n headers={\"Content-Type\": \"application/json; charset=utf-8\"},\n )\n try:\n with urllib.request.urlopen(req, timeout=self.timeout_s) as response:\n result = json.loads(response.read().decode(\"utf-8\"))\n content = result.get(\"message\", {}).get(\"content\", \"\").strip()\n if not content:\n raise RuntimeError(\"Ollama returned an empty translation\")\n return content\n except (\n urllib.error.HTTPError,\n urllib.error.URLError,\n TimeoutError,\n socket.timeout,\n json.JSONDecodeError,\n RuntimeError,\n ) as error:\n last_error = error\n if attempt >= self.max_retries:\n break\n delay = min(self.retry_backoff_s * (2 ** (attempt - 1)), 20.0)\n print(\n f\" Translation request failed ({attempt}/{self.max_retries}): \"\n f\"{error}; retrying in {delay:.1f}s\"\n )\n time.sleep(delay)\n raise RuntimeError(\n f\"Ollama translation failed after {self.max_retries} attempts: {last_error}\"\n )\n\n @staticmethod\n def _system_prompt(series_info: str = \"\") -> str:\n prompt = (\n \"你是一个轻小说翻译模型,可以流畅通顺地以日本轻小说的风格将日文翻译成\"\n \"简体中文,并联系上下文正确使用人称代词,不擅自添加原文中没有的代词。\"\n )\n if series_info:\n prompt += f\"\\n\\n系列设定与角色信息:\\n{series_info}\"\n return prompt\n\n @staticmethod\n def _user_prompt(\n texts: Sequence[str],\n glossary_terms=None,\n ) -> str:\n raw_text = \"\\n\".join(texts)\n terms = list(glossary_terms or [])\n if not terms:\n return f\"将下面的待翻译日文逐行翻译成中文:\\n{raw_text}\"\n glossary = \"\\n\".join(f\"{source}->{target}\" for source, target in terms)\n return (\n \"根据以下术语表(可以为空):\\n\"\n f\"{glossary}\\n\"\n \"将下面的待翻译日文根据对应关系和备注逐行翻译成中文:\\n\"\n f\"{raw_text}\"\n )\n\n @staticmethod\n def _context_block(context_before=None, context_after=None) -> str:\n parts = []\n if context_before:\n parts.append(\"已翻译前文(简体中文):\\n\" + \"\\n\".join(context_before))\n if context_after:\n parts.append(\"参考下文:\\n\" + \"\\n\".join(context_after))\n if not parts:\n return \"\"\n return (\n \"\\n\\n参考对话(只供理解待翻译句子的指代、语气和省略信息,\"\n \"禁止翻译、复述或输出):\\n\"\n + \"\\n\".join(parts)\n + \"\\n只输出 user 消息中待翻译日文的简体中文译文。\"\n )\n\n @staticmethod\n def _parse_lines(raw: str, expected: int) -> list[str] | None:\n raw = raw.strip()\n raw = re.sub(r\"^```(?:text|json)?\\s*\", \"\", raw, flags=re.IGNORECASE)\n raw = re.sub(r\"\\s*```$\", \"\", raw)\n if expected == 1:\n return [re.sub(r\"\\s+\", \" \", raw).strip()] if raw else None\n lines = [line.strip() for line in raw.splitlines() if line.strip()]\n if len(lines) == expected + 1 and lines[0].rstrip(\"::\") in {\n \"翻译\",\n \"译文\",\n \"翻译结果\",\n }:\n lines = lines[1:]\n if len(lines) != expected:\n return None\n if all(re.match(r\"^\\d+[.、]\\s*\", line) for line in lines):\n lines = [re.sub(r\"^\\d+[.、]\\s*\", \"\", line) for line in lines]\n return lines\n\n @staticmethod\n def _tagged_texts(texts: Sequence[str]) -> list[str]:\n \"\"\"Attach stable IDs so a model cannot silently swap batch lines.\"\"\"\n return [f\"[[L{index:03d}]] {text}\" for index, text in enumerate(texts)]\n\n @staticmethod\n def _parse_tagged_lines(raw: str, expected: int) -> list[str] | None:\n \"\"\"Parse tagged translations, accepting brackets commonly normalized by LLMs.\"\"\"\n raw = raw.strip()\n raw = re.sub(r\"^```(?:text|json)?\\s*\", \"\", raw, flags=re.IGNORECASE)\n raw = re.sub(r\"\\s*```$\", \"\", raw)\n tagged = {}\n pattern = re.compile(\n r\"^\\s*(?:\\[\\[|\\[|【)\\s*L(\\d{1,4})\\s*(?:\\]\\]|\\]|】)\\s*(.*?)\\s*$\"\n )\n for line in raw.splitlines():\n if not line.strip():\n continue\n match = pattern.match(line)\n if not match:\n return None\n index = int(match.group(1))\n target = match.group(2).strip()\n if index >= expected or index in tagged or not target:\n return None\n tagged[index] = target\n if set(tagged) != set(range(expected)):\n return None\n return [tagged[index] for index in range(expected)]\n\n @staticmethod\n def _is_nonverbal_source(source: str) -> bool:\n \"\"\"Return whether a cue contains only vocal markers and punctuation.\"\"\"\n vocal_markers = {\"っ\", \"ッ\", \"ー\"}\n return bool(source) and all(\n character in vocal_markers\n or unicodedata.category(character)[0] in {\"P\", \"S\", \"Z\"}\n for character in source\n )\n\n @staticmethod\n def _valid_translation(source: str, target: str) -> bool:\n if not target or target.startswith(\"[API Error\"):\n return False\n if any(\n marker in target\n for marker in (\n \"将下面\",\n \"术语表\",\n \"翻译结果如下\",\n \"上文:\",\n \"上文:\",\n \"下文:\",\n \"下文:\",\n \"参考对话\",\n \"只供理解\",\n \"只输出 user 消息\",\n \"待翻译日文的简体中文译文\",\n )\n ):\n return False\n nonverbal_source = OllamaAdapter._is_nonverbal_source(source)\n if (\n not re.search(r\"[\\u3400-\\u9fffA-Za-z0-9]\", target)\n and not nonverbal_source\n ):\n return False\n if len(target) > max(40, len(source) * 5):\n return False\n if re.search(r\"(.)\\1{15,}\", target):\n return False\n if OllamaAdapter._has_suspicious_repeated_phrase(source, target):\n return False\n has_kana = bool(re.search(r\"[\\u3040-\\u30ff]\", source))\n if has_kana and target.strip() == source.strip():\n return False\n return True\n\n @staticmethod\n def _has_suspicious_repeated_phrase(source: str, target: str) -> bool:\n \"\"\"Detect three merged neighboring lines with a repeated Chinese lead-in.\"\"\"\n compact_target = re.sub(r\"[^\\u3400-\\u9fffA-Za-z0-9]\", \"\", target)\n compact_source = re.sub(r\"\\s+\", \"\", source)\n source_has_triplet = bool(re.search(r\"(.{1,4})\\1\\1\", compact_source))\n if source_has_triplet:\n return False\n for size in range(8, 2, -1):\n for start in range(0, max(0, len(compact_target) - size + 1)):\n phrase = compact_target[start:start + size]\n if compact_target.count(phrase) >= 3:\n return True\n return False\n\n def translate_batch(\n self,\n texts: Sequence[str],\n glossary_terms=None,\n context_before=None,\n context_after=None,\n ) -> list[str]:\n texts = [str(text).strip() for text in texts]\n if not texts:\n return []\n glossary_terms = self._matching_terms(texts, glossary_terms)\n context_enabled = context_before is not None or context_after is not None\n if context_enabled and len(texts) > 1:\n translated = []\n outer_before = list(context_before or [])\n outer_after = list(context_after or [])\n for index, text in enumerate(texts):\n translated.extend(\n self.translate_batch(\n [text],\n self._matching_terms([text], glossary_terms),\n context_before=outer_before + texts[:index],\n context_after=texts[index + 1:] + outer_after,\n )\n )\n return translated\n tagged_batch = len(texts) > 1\n prompt_texts = self._tagged_texts(texts) if tagged_batch else texts\n system_prompt = self._system_prompt(self.series_info) + self._context_block(\n context_before,\n context_after,\n )\n if tagged_batch:\n system_prompt += (\n \"\\n逐行翻译。每行开头的 [[Lnnn]] 是不可翻译的行号;译文必须保留\"\n \"相同行号,不得交换、合并或遗漏。\"\n )\n messages = [\n {\"role\": \"system\", \"content\": system_prompt},\n {\n \"role\": \"user\",\n \"content\": self._user_prompt(prompt_texts, glossary_terms),\n },\n ]\n try:\n raw = self._call(\n messages,\n num_predict=min(4096, max(512, len(texts) * 128)),\n )\n except Exception as error:\n if len(texts) == 1:\n return [\n self._fallback_invalid_single(\n texts[0],\n glossary_terms,\n f\"primary translation request failed: {error}\",\n context_before=context_before,\n context_after=context_after,\n )\n ]\n # A batch-level transport failure may be request-size-specific. Split\n # it before allowing each individual line to use its fallback chain.\n middle = len(texts) // 2\n return self.translate_batch(\n texts[:middle],\n glossary_terms,\n context_before=context_before,\n context_after=texts[middle:] + list(context_after or []),\n ) + self.translate_batch(\n texts[middle:],\n glossary_terms,\n context_before=list(context_before or []) + texts[:middle],\n context_after=context_after,\n )\n translated = (\n self._parse_tagged_lines(raw, len(texts))\n if tagged_batch\n else self._parse_lines(raw, len(texts))\n )\n if translated and all(\n self._valid_translation(source, target)\n for source, target in zip(texts, translated)\n ):\n for source in texts:\n self._translation_models[source] = self.model\n self._translation_fallbacks[source] = False\n return translated\n\n if len(texts) == 1:\n return [\n self._recover_invalid_single(\n texts[0],\n glossary_terms,\n raw,\n context_before=context_before,\n context_after=context_after,\n )\n ]\n # A model may occasionally merge or drop a line. Recursively reduce the\n # batch instead of guessing which result belongs to which timestamp.\n middle = len(texts) // 2\n return self.translate_batch(\n texts[:middle],\n glossary_terms,\n context_before=context_before,\n context_after=texts[middle:] + list(context_after or []),\n ) + self.translate_batch(\n texts[middle:],\n glossary_terms,\n context_before=list(context_before or []) + texts[:middle],\n context_after=context_after,\n )\n\n def _recover_invalid_single(\n self,\n source: str,\n glossary_terms,\n raw: str,\n context_before=None,\n context_after=None,\n ) -> str:\n \"\"\"Retry a rejected line with a constrained prompt before using a fallback.\"\"\"\n last_raw = raw\n for attempt in range(1, self.validation_retries + 1):\n messages = [\n {\n \"role\": \"system\",\n \"content\": (\n self._system_prompt(self.series_info)\n + self._context_block(context_before, context_after)\n + \"\\n只输出一行简洁的简体中文译文;禁止解释、重复、引号包裹和遗漏。\"\n ),\n },\n {\n \"role\": \"user\",\n \"content\": (\n \"上一次译文未通过字幕校验。请重新翻译且只输出一行:\\n\"\n + self._user_prompt([source], glossary_terms)\n ),\n },\n ]\n try:\n last_raw = self._call(messages, num_predict=256)\n except Exception as error:\n last_raw = (\n f\"validation retry {attempt}/{self.validation_retries} \"\n f\"request failed: {error}\"\n )\n print(f\" Translation {last_raw}\")\n break\n parsed = self._parse_lines(last_raw, 1)\n if parsed and self._valid_translation(source, parsed[0]):\n self._translation_models[source] = self.model\n self._translation_fallbacks[source] = False\n print(\n f\" Translation validation recovered on retry \"\n f\"{attempt}/{self.validation_retries}\"\n )\n return parsed[0]\n return self._fallback_invalid_single(\n source,\n glossary_terms,\n last_raw,\n context_before=context_before,\n context_after=context_after,\n )\n\n def _fallback_invalid_single(\n self,\n source: str,\n glossary_terms,\n raw: str,\n context_before=None,\n context_after=None,\n ) -> str:\n raise RuntimeError(\n f\"{self.name()} returned an invalid single-line translation after \"\n f\"{self.validation_retries} validation retries: {raw[:200]}\"\n )\n\n @staticmethod\n def _matching_terms(texts: Sequence[str], glossary_terms=None) -> list[tuple[str, str]]:\n joined = \"\\n\".join(texts)\n matches = [\n (source, target)\n for source, target in (glossary_terms or [])\n if source in joined\n ]\n return sorted(matches, key=lambda item: len(item[0]), reverse=True)\n\n def translate(self, text, context_before=None, context_after=None):\n return self.translate_batch(\n [text],\n context_before=context_before,\n context_after=context_after,\n )[0]\n\n\n# ============ Sakura ============\n\nclass SakuraAdapter(OllamaAdapter):\n def __init__(self, config: dict):\n super().__init__(config, \"sakura\")\n sakura_config = config.get(\"sakura\", {})\n fallback_name = sakura_config.get(\n \"validation_fallback_backend\", \"galtransl\"\n )\n fallback_config = config.get(fallback_name, {})\n self.fallback_adapter = None\n if fallback_name == \"galtransl\" and fallback_config.get(\"model\"):\n self.fallback_adapter = GalTranslAdapter(config)\n self.rescue_models = tuple(sakura_config.get(\"validation_rescue_models\", []))\n self.rescue_base_url = str(\n sakura_config.get(\"validation_rescue_base_url\", \"\")\n ).rstrip(\"/\")\n self.rescue_api_key_file = str(\n sakura_config.get(\"validation_rescue_api_key_file\", \"\")\n )\n self.rescue_attempts = max(\n 1, int(sakura_config.get(\"validation_rescue_attempts\", 2))\n )\n self.quarantine_on_failure = bool(\n sakura_config.get(\"validation_quarantine_on_failure\", True)\n )\n\n def _rescue_invalid_single(\n self,\n source: str,\n glossary_terms,\n context_before=None,\n context_after=None,\n ) -> str | None:\n \"\"\"Use the rotating SenseNova accounts only after both local models fail.\"\"\"\n if not self.rescue_models or not self.rescue_base_url:\n return None\n # Import lazily so local-only translation has no review-provider dependency.\n from scripts.review_agents import model_chat\n\n messages = [\n {\n \"role\": \"system\",\n \"content\": (\n self._system_prompt(self.series_info)\n + self._context_block(context_before, context_after)\n + \"\\n本地翻译模型未通过校验。只输出一行简洁准确的简体中文译文;\"\n \"禁止解释、引号包裹、重复或输出日文。\"\n ),\n },\n {\n \"role\": \"user\",\n \"content\": self._user_prompt([source], glossary_terms),\n },\n ]\n for model in self.rescue_models:\n for attempt in range(1, self.rescue_attempts + 1):\n try:\n raw = model_chat(\n messages,\n provider=\"openai\",\n base_url=self.rescue_base_url,\n api_key_file=self.rescue_api_key_file,\n model=model,\n json_mode=False,\n temperature=0.1,\n timeout_s=self.timeout_s,\n )\n except Exception as error:\n print(\n f\" Translation rescue {model} failed \"\n f\"({attempt}/{self.rescue_attempts}): {error}\"\n )\n continue\n parsed = self._parse_lines(raw, 1)\n if parsed and self._valid_translation(source, parsed[0]):\n self._translation_models[source] = model\n self._translation_fallbacks[source] = True\n self._translation_errors.pop(source, None)\n print(f\" Translation validation rescue: {self.model} -> {model}\")\n return parsed[0]\n print(\n f\" Translation rescue {model} returned invalid output \"\n f\"({attempt}/{self.rescue_attempts})\"\n )\n return None\n\n def _quarantine_invalid_single(self, source: str, reason: str) -> str:\n \"\"\"Preserve a failed line for downstream AI review without poisoning TM.\"\"\"\n if not self.quarantine_on_failure:\n raise RuntimeError(reason)\n self._translation_models[source] = \"untranslated-source-quarantine\"\n self._translation_fallbacks[source] = True\n self._translation_errors[source] = reason[:1000]\n print(\n \" WARNING: translation quarantined for multi-agent/MQM review: \"\n f\"{source[:80]}\"\n )\n return source\n\n def _fallback_invalid_single(\n self,\n source: str,\n glossary_terms,\n raw: str,\n context_before=None,\n context_after=None,\n ) -> str:\n fallback_error = \"configured GalTransl fallback is unavailable\"\n if self.fallback_adapter is not None:\n self.fallback_adapter.series_info = self.series_info\n try:\n target = _call_batch_with_optional_context(\n self.fallback_adapter,\n [source],\n glossary_terms,\n context_before=context_before,\n context_after=context_after,\n )[0]\n except Exception as error:\n fallback_error = str(error)\n else:\n self._translation_models[source] = self.fallback_adapter.result_model(source)\n self._translation_fallbacks[source] = True\n self._translation_errors.pop(source, None)\n print(\n f\" Translation validation fallback: {self.model} -> \"\n f\"{self._translation_models[source]}\"\n )\n return target\n\n rescued = self._rescue_invalid_single(\n source,\n glossary_terms,\n context_before=context_before,\n context_after=context_after,\n )\n if rescued is not None:\n return rescued\n reason = (\n \"Sakura validation retries, configured GalTransl fallback, and \"\n f\"SenseNova rescue all failed; GalTransl error: {fallback_error}; \"\n f\"last Sakura output: {raw[:200]}\"\n )\n return self._quarantine_invalid_single(source, reason)\n\n def name(self):\n return f\"Sakura ({self.model})\"\n\n\n# ============ Qwen ============\n\nclass QwenAdapter(OllamaAdapter):\n def __init__(self, config: dict):\n super().__init__(config, \"qwen\")\n\n def name(self):\n return f\"Qwen ({self.model})\"\n\n\n# ============ GalTransl ============\n\nclass GalTranslAdapter(OllamaAdapter):\n def __init__(self, config: dict):\n super().__init__(config, \"galtransl\")\n\n def name(self):\n return f\"GalTransl ({self.model})\"\n\n\n# ============ External API ============\n\nclass ExternalAdapter(TranslatorAdapter):\n \"\"\"OpenAI-compatible API adapter.\"\"\"\n\n def __init__(self, config: dict):\n super().__init__(config)\n ext = config.get(\"external\", {})\n self.api_url = ext.get(\"api_url\", \"\")\n self.api_key = ext.get(\"api_key\", \"\")\n self.model = ext.get(\"model\", \"\")\n\n def name(self):\n return f\"External ({self.model})\"\n\n def translate(self, text, context_before=None, context_after=None):\n if not self.api_url:\n return \"[External API not configured]\"\n\n headers = {\n \"Content-Type\": \"application/json\",\n \"Authorization\": f\"Bearer {self.api_key}\",\n }\n payload = {\n \"model\": self.model,\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": self._external_system_prompt()\n + OllamaAdapter._context_block(context_before, context_after),\n },\n {\n \"role\": \"user\",\n \"content\": OllamaAdapter._user_prompt(\n [text],\n ),\n },\n ],\n \"temperature\": 0.1,\n }\n data = json.dumps(payload).encode(\"utf-8\")\n req = urllib.request.Request(\n self.api_url, data=data, headers=headers,\n )\n try:\n resp = urllib.request.urlopen(req, timeout=120)\n result = json.loads(resp.read().decode(\"utf-8\"))\n return result.get(\"choices\", [{}])[0].get(\"message\", {}).get(\"content\", \"\").strip()\n except Exception as e:\n return f\"[API Error: {e}]\"\n\n def _external_system_prompt(self) -> str:\n prompt = \"你是一个轻小说翻译模型,可以将日语翻译成简体中文。\"\n if self.series_info:\n prompt += f\"\\n\\n系列设定与角色信息:\\n{self.series_info}\"\n return prompt\n\n\ndef _ensure_builtin_translator_plugins() -> None:\n for name, adapter_class in {\n \"sakura\": SakuraAdapter,\n \"qwen\": QwenAdapter,\n \"galtransl\": GalTranslAdapter,\n \"external\": ExternalAdapter,\n }.items():\n plugin_registry.register_if_missing(\n \"translator\",\n name,\n adapter_class,\n source=\"builtin:translator_adapter\",\n description=f\"Built-in {adapter_class.__name__}\",\n )\n\n\n# ============ Config ============\n\ndef load_config(path: str = \"\") -> dict:\n \"\"\"Load translator config from file, or return default.\"\"\"\n if path:\n config_path = Path(path)\n if config_path.exists():\n with open(config_path, encoding=\"utf-8\") as f:\n return json.load(f)\n return dict(DEFAULT_CONFIG)\n\n\ndef save_default_config(path: str = \"translator_config.json\"):\n \"\"\"Save default config to file.\"\"\"\n config_path = Path(path)\n with open(config_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(DEFAULT_CONFIG, f, ensure_ascii=False, indent=2)\n print(f\"Default config saved: {path}\")\n\n\n# ============ Evaluate ============\n\ndef evaluate():\n print(\"\\n============================================================\")\n print(\"S14.1 TRANSLATOR ADAPTER EVALUATION\")\n print(\"============================================================\")\n\n config = load_config()\n\n test_texts = [\n \"おはよう、唯\",\n \"今日も元気だね\",\n \"澪ちゃん、一緒に練習しない?\",\n ]\n\n backends_to_test = [\"sakura\", \"qwen\", \"galtransl\"]\n results = {}\n\n for backend_name in backends_to_test:\n cfg = dict(config)\n cfg[\"backend\"] = backend_name\n try:\n adapter = TranslatorAdapter.from_config(cfg)\n print(f\"\\n--- Backend: {adapter.name()} ---\")\n\n backend_results = []\n for text in test_texts:\n t0 = time.time()\n try:\n result = adapter.translate(text)\n elapsed = time.time() - t0\n except Exception as e:\n result = f\"[ERROR: {e}]\"\n elapsed = time.time() - t0\n backend_results.append({\n \"ja\": text,\n \"zh\": result,\n \"time_s\": round(elapsed, 2),\n })\n print(f\" {text[:20]} -> {result[:30]} ({elapsed:.1f}s)\")\n\n results[backend_name] = {\n \"name\": adapter.name(),\n \"samples\": backend_results,\n }\n except Exception as e:\n print(f\" SKIP: {e}\")\n results[backend_name] = {\"name\": backend_name, \"error\": str(e)}\n\n # Save\n out_path = project_root / \"docs\" / \"evaluation\" / \"S14.1_adapter_results.json\"\n with open(out_path, \"w\", encoding=\"utf-8\") as f:\n json.dump({\n \"config\": {k: v for k, v in config.items() if k != \"external\"},\n \"results\": results,\n }, f, ensure_ascii=False, indent=2)\n print(f\"\\nResults saved: {out_path}\")\n\n print(\"\\n============================================================\")\n print(\"EVALUATION COMPLETE\")\n print(\"============================================================\")\n\n\n# ============ CLI ============\n\ndef main():\n parser = argparse.ArgumentParser(description=\"S14.1 Translator Adapter\")\n parser.add_argument(\"--text\", type=str, help=\"Japanese text to translate\")\n parser.add_argument(\"--backend\", type=str, help=\"Translation backend or plugin name\")\n parser.add_argument(\"--config\", type=str, default=\"\", help=\"Config file path\")\n parser.add_argument(\"--plugin\", action=\"append\", default=[],\n help=\"Trusted local plugin .py file (repeatable)\")\n parser.add_argument(\"--list-backends\", action=\"store_true\", help=\"List available backends\")\n parser.add_argument(\"--save-config\", type=str, help=\"Save default config to file\")\n parser.add_argument(\"--evaluate\", action=\"store_true\")\n args = parser.parse_args()\n\n load_plugins(args.plugin)\n _ensure_builtin_translator_plugins()\n\n if args.list_backends:\n print(\"Available backends:\")\n for spec in plugin_registry.specs(\"translator\"):\n print(f\" {spec['name']:<12} {spec['description']} [{spec['source']}]\")\n return\n\n if args.save_config:\n save_default_config(args.save_config)\n return\n\n if args.evaluate:\n evaluate()\n return\n\n if args.text:\n config = load_config(args.config)\n if args.backend:\n config[\"backend\"] = args.backend\n adapter = TranslatorAdapter.from_config(config)\n result = adapter.translate(args.text)\n print(result)\n return\n\n parser.print_help()\n\n\nif __name__ == \"__main__\":\n main()\n"}}, "config/quality_mqm.sensenova.json": {"4d507921cd926e659e5a5a22d58eec34686302b8": {"t": 1785768284.0659661, "n": 580, "h": "4e97e01b0de54431", "c": "{\n \"provider\": \"openai\",\n \"base_url\": \"https://token.sensenova.cn/v1\",\n \"api_key_file\": \"config/sensenova_apikeys\",\n \"judge_models\": [\n \"sensenova-6.7-flash-lite\",\n \"deepseek-v4-flash\"\n ],\n \"judge_fallback_models\": {\n \"sensenova-6.7-flash-lite\": \"glm-5.2\"\n },\n \"editor_model\": \"sensenova-6.7-flash-lite\",\n \"editor_fallback_model\": \"glm-5.2\",\n \"max_workers\": 2,\n \"context_window\": 5,\n \"review_threshold\": 80,\n \"min_judge_confidence\": 0.75,\n \"min_editor_confidence\": 0.85,\n \"min_final_score\": 80,\n \"min_improvement\": 5,\n \"retries\": 6,\n \"timeout_s\": 600\n}\n"}}}}