1 line
48 KiB
JSON
1 line
48 KiB
JSON
{"repo": "azumag/doci", "files": {"doci/ai_text.py": {"bab0f0cbaa99f9113fbd08c9aefec32b65664f3b": {"t": 1785596355.2225454, "n": 40495, "h": "2378ed137b6f8504", "c": "\"\"\"台本生成（OpenCode Go / qwen3.7-plus）。\n\nMinimax は文章生成に使わない（方針）。\nバックエンド:\n - claude_cli (旧・明示時のみ): 認証済みの `claude` CLI を print モードで呼ぶ\n - anthropic (クラウド): Anthropic API (ANTHROPIC_API_KEY) を直叩き\n - opencode (代替): `opencode run --agent ...`\n\"\"\"\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport queue\nimport re\nimport socket\nimport subprocess\nimport threading\nfrom collections.abc import Callable\nimport sys\nimport time\nimport urllib.error\nimport urllib.request\nfrom datetime import date as _date\nfrom pathlib import Path\n\nfrom . import channel, config, corners, llm\nfrom .channel import ChannelSpec, CornerSpec\n\nREQUIRED_KEYS = (\"title\", \"description\", \"tags\", \"narration\", \"scenes\")\n_DEFAULT_WRITE_TIMEOUT = object()\n_RESPONSE_END = object()\n\n\ndef _shutdown_response(response) -> None: # type: ignore[no-untyped-def]\n \"\"\"読み取り待ちのソケットを、可能ならshutdownしてワーカーを解放する。\"\"\"\n for candidate in (\n getattr(response, \"fp\", None),\n getattr(getattr(response, \"fp\", None), \"raw\", None),\n getattr(getattr(getattr(response, \"fp\", None), \"raw\", None), \"_sock\", None),\n ):\n shutdown = getattr(candidate, \"shutdown\", None)\n if callable(shutdown):\n try:\n shutdown(socket.SHUT_RDWR)\n except (OSError, TypeError):\n pass\n\n\nclass _ResponseReadWorker:\n \"\"\"settimeout非対応の応答を、接続ごとに1本の読み取りワーカーで消費する。\"\"\"\n\n def __init__(\n self,\n reader: Callable[[], object],\n response, # type: ignore[no-untyped-def]\n *,\n empty_is_end: bool = False,\n ) -> None:\n self._reader = reader\n self._response = response\n self._empty_is_end = empty_is_end\n self._events: queue.Queue[tuple[bool, object]] = queue.Queue(maxsize=1)\n self._cancelled = threading.Event()\n self._worker = threading.Thread(\n target=self._run, daemon=True, name=\"doci-response-reader\"\n )\n self._worker.start()\n\n def _publish(self, event: tuple[bool, object]) -> bool:\n while not self._cancelled.is_set():\n try:\n self._events.put(event, timeout=0.1)\n return True\n except queue.Full:\n continue\n return False\n\n def _run(self) -> None:\n while not self._cancelled.is_set():\n try:\n value = self._reader()\n except StopIteration:\n self._publish((False, _RESPONSE_END))\n return\n except BaseException as exc: # noqa: BLE001 - propagate reader errors\n self._publish((False, exc))\n return\n if not self._publish((True, value)):\n return\n if self._empty_is_end and not value:\n return\n\n def next(self, timeout_seconds: float) -> object:\n try:\n succeeded, value = self._events.get(timeout=max(0.001, timeout_seconds))\n except queue.Empty as exc:\n self.cancel()\n raise socket.timeout(\"stream read timeout\") from exc\n if succeeded:\n return value\n if value is _RESPONSE_END:\n raise StopIteration\n raise value # type: ignore[misc]\n\n def cancel(self) -> None:\n if self._cancelled.is_set():\n return\n self._cancelled.set()\n _shutdown_response(self._response)\n close = getattr(self._response, \"close\", None)\n if callable(close):\n try:\n close()\n except OSError:\n pass\n self._worker.join(timeout=0.2)\n\n\ndef _read_response_until_deadline(response, deadline: float) -> bytes: # type: ignore[no-untyped-def]\n \"\"\"settimeout非対応のHTTP応答を、接続単位のworkerで期限まで読む。\"\"\"\n chunks: list[bytes] = []\n response_timeout_supported = False\n fallback_reader: _ResponseReadWorker | None = None\n try:\n while True:\n remaining = deadline - time.monotonic()\n if remaining <= 0:\n raise TimeoutError(\"Anthropic API が時間上限に達しました\")\n for candidate in (\n getattr(response, \"fp\", None),\n getattr(getattr(response, \"fp\", None), \"raw\", None),\n getattr(getattr(getattr(response, \"fp\", None), \"raw\", None), \"_sock\", None),\n ):\n setter = getattr(candidate, \"settimeout\", None)\n if callable(setter):\n try:\n setter(remaining)\n except OSError:\n pass\n else:\n response_timeout_supported = True\n try:\n if fallback_reader is not None:\n chunk = fallback_reader.next(remaining)\n elif response_timeout_supported:\n chunk = response.read(4096)\n else:\n fallback_reader = _ResponseReadWorker(\n lambda: response.read(4096), response, empty_is_end=True\n )\n chunk = fallback_reader.next(remaining)\n except socket.timeout as exc:\n raise TimeoutError(\"Anthropic API が時間上限に達しました\") from exc\n if not chunk:\n return b\"\".join(chunks)\n chunks.append(chunk) # type: ignore[arg-type]\n finally:\n if fallback_reader is not None:\n fallback_reader.cancel()\n\n\ndef _monotonic() -> float:\n \"\"\"差し替え可能な時計。全体予算のテストが標準timeを汚染しないようにする。\"\"\"\n return time.monotonic()\n\n# 互換用エイリアス（JSON抽出/CLI実行は共通モジュール llm に集約）\n_extract_json = llm.extract_json\n\n\ndef _write_timeout(\n override: int | float | None | object = _DEFAULT_WRITE_TIMEOUT,\n) -> int | float | None:\n \"\"\"CLI/APIの全体待機上限。0は明示的な長文待機モード。\"\"\"\n if override is _DEFAULT_WRITE_TIMEOUT:\n return _whole_write_timeout()\n if override is None:\n return None\n return override if override > 0 else None\n\n\ndef _whole_write_timeout() -> int | None:\n \"\"\"ストリーム全体の上限。0は長文を最後まで待つ。\"\"\"\n return config.WRITE_LLM_TIMEOUT if config.WRITE_LLM_TIMEOUT > 0 else None\n\n\ndef _run_claude_cli(\n prompt: str,\n model: str,\n timeout: int | float | None | object = _DEFAULT_WRITE_TIMEOUT,\n) -> str:\n return llm.run_claude(\n prompt, config.legacy_claude_model(model), timeout=_write_timeout(timeout)\n )\n\n\ndef _run_anthropic(\n prompt: str,\n model: str,\n timeout: int | float | None | object = _DEFAULT_WRITE_TIMEOUT,\n) -> str:\n key = config.get(\"ANTHROPIC_API_KEY\")\n if not key:\n raise RuntimeError(\"ANTHROPIC_API_KEY が未設定です (TEXT_BACKEND=anthropic)\")\n body = json.dumps(\n {\n \"model\": config.legacy_claude_model(model),\n \"max_tokens\": 2000,\n \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n }\n ).encode(\"utf-8\")\n req = urllib.request.Request(\n \"https://api.anthropic.com/v1/messages\",\n data=body,\n headers={\n \"x-api-key\": key,\n \"anthropic-version\": \"2023-06-01\",\n \"content-type\": \"application/json\",\n },\n )\n request_timeout = _write_timeout(timeout)\n deadline = (\n time.monotonic() + request_timeout if request_timeout is not None else None\n )\n open_timeout = request_timeout\n if deadline is not None:\n open_timeout = max(0.001, deadline - time.monotonic())\n with urllib.request.urlopen(req, timeout=open_timeout) as resp:\n payload = resp.read() if deadline is None else _read_response_until_deadline(resp, deadline)\n data = json.loads(payload.decode(\"utf-8\"))\n return \"\".join(b.get(\"text\", \"\") for b in data.get(\"content\", []))\n\n\ndef _opencode_go_key() -> str:\n \"\"\"環境変数を優先し、無ければ OpenCode の既存認証ストアから Go の鍵を読む。\"\"\"\n if config.OPENCODE_GO_API_KEY:\n return config.OPENCODE_GO_API_KEY\n auth_file = Path(config.OPENCODE_AUTH_FILE).expanduser()\n try:\n auth = json.loads(auth_file.read_text(encoding=\"utf-8\"))\n key = auth.get(\"opencode-go\", {}).get(\"key\", \"\")\n except (OSError, json.JSONDecodeError, AttributeError):\n key = \"\"\n if not key:\n raise RuntimeError(\n \"OPENCODE_GO_API_KEY が未設定で、OpenCode認証ストアにも \"\n f\"opencode-go の鍵がありません: {auth_file}\"\n )\n return key\n\n\ndef _opencode_go_model(model: str) -> str:\n \"\"\"OpenCode Go APIへ渡せるモデル名だけを許可する。\n\n bare model はゲートウェイの既定プロバイダとして後方互換に受け入れるが、\n provider-qualified な別プロバイダ名をそのまま送ると、Go APIではなく別の\n 認証経路を暗黙に要求するため、既定のQwenへ安全に戻す。\n \"\"\"\n if not model:\n raise RuntimeError(\n \"TEXT_BACKEND=opencode_go ではモデルを指定してください \"\n f\"（例: {config.OPENCODE_GO_DEFAULT_MODEL}）\"\n )\n provider, separator, _ = model.partition(\"/\")\n if separator and provider != \"opencode-go\":\n raise RuntimeError(\n \"TEXT_BACKEND=opencode_go では OPENCODE_MODEL を \"\n \"opencode-go/<model> 形式で指定してください\"\n )\n model_id = model.split(\"/\", 1)[1] if separator else model\n if model_id.startswith((\"claude-\", \"anthropic/\")):\n raise RuntimeError(\n \"TEXT_BACKEND=opencode_go ではClaudeモデルを使えません。 \"\n f\"{config.OPENCODE_GO_DEFAULT_MODEL} を指定してください\"\n )\n return model\n\n\n_DEFAULT_OPENCODE_GO_TIMEOUT = object()\n\n\ndef _run_opencode_go(\n prompt: str,\n model: str,\n timeout: int | None | object = _DEFAULT_OPENCODE_GO_TIMEOUT,\n) -> str:\n \"\"\"OpenCode CLIを介さず、OpenCode GoのAnthropic互換APIへ直接接続する。\"\"\"\n model = _opencode_go_model(model)\n _, sep, model_id = model.partition(\"/\")\n if not sep:\n model_id = model\n body = json.dumps(\n {\n \"model\": model_id,\n \"max_tokens\": config.OPENCODE_GO_MAX_TOKENS,\n \"stream\": True,\n \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n }\n ).encode(\"utf-8\")\n req = urllib.request.Request(\n f\"{config.OPENCODE_GO_BASE_URL}/messages\",\n data=body,\n headers={\n \"x-api-key\": _opencode_go_key(),\n \"anthropic-version\": \"2023-06-01\",\n \"content-type\": \"application/json\",\n # Python標準UAはGoゲートウェイで403になるため、アプリ固有UAを明示する。\n \"user-agent\": \"doci/1.0\",\n },\n )\n started = time.monotonic()\n next_progress = 60.0\n text_parts: list[str] = []\n text_chars = 0\n stop_reason = \"\"\n if timeout is _DEFAULT_OPENCODE_GO_TIMEOUT:\n deadline_timeout = _whole_write_timeout()\n else:\n deadline_timeout = timeout if isinstance(timeout, (int, float)) and timeout > 0 else None\n # 全体上限とidle上限を同時に0にすると、Claudeフォールバックなしの既定経路が\n # 無音SSEで復帰不能になるため、明示的な全体無制限時にもidleだけは残す。\n idle_timeout = (\n config.WRITE_LLM_IDLE_TIMEOUT\n if config.WRITE_LLM_IDLE_TIMEOUT > 0\n else (300 if deadline_timeout is None else None)\n )\n request_limits = [value for value in (deadline_timeout, idle_timeout) if value is not None]\n request_timeout = min(request_limits) if request_limits else None\n deadline_expired = threading.Event()\n received_terminal = False\n\n def set_stream_timeout(response, timeout_seconds: float | None) -> bool: # type: ignore[no-untyped-def]\n \"\"\"各行の読み取り前にソケットへ残り時間を設定し、全体上限をreadlineにも適用する。\"\"\"\n if timeout_seconds is None:\n return True\n candidates = (\n getattr(getattr(response, \"fp\", None), \"raw\", None),\n getattr(getattr(getattr(response, \"fp\", None), \"raw\", None), \"_sock\", None),\n )\n for candidate in candidates:\n setter = getattr(candidate, \"settimeout\", None)\n if callable(setter):\n try:\n setter(timeout_seconds)\n except OSError:\n pass\n else:\n return True\n return False\n\n try:\n with urllib.request.urlopen(req, timeout=request_timeout) as resp:\n fallback_reader: _ResponseReadWorker | None = None\n try:\n stream = iter(resp)\n stream_timeout_warning_logged = False\n response_read = getattr(resp, \"read1\", None) or getattr(resp, \"read\", None)\n byte_mode = callable(response_read)\n line_buffer = bytearray()\n last_socket_timeout: float | None = None\n while True:\n remaining = (\n deadline_timeout - (time.monotonic() - started)\n if deadline_timeout is not None\n else None\n )\n if remaining is not None and remaining <= 0:\n deadline_expired.set()\n raise RuntimeError(\"OpenCode Go API が時間上限に達しました\")\n if remaining is not None:\n remaining = max(0.001, remaining)\n read_timeout = remaining\n if idle_timeout is not None:\n read_timeout = (\n min(read_timeout, idle_timeout)\n if read_timeout is not None\n else idle_timeout\n )\n update_socket_timeout = (\n read_timeout is not None\n and (\n last_socket_timeout is None\n or abs(read_timeout - last_socket_timeout) >= 0.1\n )\n )\n if read_timeout is None:\n stream_timeout_applied = True\n elif update_socket_timeout:\n stream_timeout_applied = set_stream_timeout(resp, read_timeout)\n else:\n stream_timeout_applied = last_socket_timeout is not None\n if update_socket_timeout and stream_timeout_applied:\n last_socket_timeout = read_timeout\n if not stream_timeout_applied:\n if not stream_timeout_warning_logged:\n _log(\"警告: OpenCode Goストリームのソケット期限を設定できません\")\n stream_timeout_warning_logged = True\n try:\n if b\"\\n\" in line_buffer:\n raw_line, _, remainder = line_buffer.partition(b\"\\n\")\n line_buffer = bytearray(remainder)\n raw = raw_line + b\"\\n\"\n elif byte_mode:\n if fallback_reader is not None:\n chunk = fallback_reader.next(read_timeout)\n elif stream_timeout_applied:\n chunk = response_read(4096)\n else:\n fallback_reader = _ResponseReadWorker(\n lambda: response_read(4096), resp\n )\n chunk = fallback_reader.next(read_timeout)\n if not chunk:\n if line_buffer:\n raw = bytes(line_buffer)\n line_buffer.clear()\n else:\n raise StopIteration\n else:\n line_buffer.extend(chunk)\n continue\n elif fallback_reader is not None:\n raw = fallback_reader.next(read_timeout)\n elif stream_timeout_applied:\n raw = next(stream)\n else:\n fallback_reader = _ResponseReadWorker(\n lambda: next(stream), resp\n )\n raw = fallback_reader.next(read_timeout)\n except socket.timeout as exc:\n if (\n deadline_timeout is not None\n and time.monotonic() - started >= deadline_timeout\n ):\n deadline_expired.set()\n raise RuntimeError(\"OpenCode Go API が時間上限に達しました\") from exc\n raise RuntimeError(\"OpenCode Go API の無通信タイムアウト\") from exc\n except StopIteration:\n if line_buffer:\n raw = bytes(line_buffer)\n line_buffer.clear()\n else:\n if (\n deadline_expired.is_set()\n or (\n deadline_timeout is not None\n and time.monotonic() - started >= deadline_timeout\n )\n ):\n deadline_expired.set()\n raise RuntimeError(\n \"OpenCode Go API が時間上限に達しました\"\n )\n # [DONE]/stop_reasonを省略するゲートウェイでも、ストリームの\n # 自然終了は完全な本文の終端として受け入れる。\n received_terminal = True\n break\n line = raw.decode(\"utf-8\", errors=\"replace\").strip()\n if not line.startswith(\"data:\"):\n continue\n payload = line[5:].lstrip()\n if not payload:\n continue\n if payload == \"[DONE]\":\n received_terminal = True\n break\n try:\n event = json.loads(payload)\n except json.JSONDecodeError:\n continue\n event_type = event.get(\"type\")\n if event_type == \"error\":\n raise RuntimeError(f\"OpenCode Go API error: {event.get('error', event)}\")\n delta = event.get(\"delta\") or {}\n if delta.get(\"type\") == \"text_delta\":\n part = delta.get(\"text\", \"\")\n text_parts.append(part)\n text_chars += len(part)\n if event_type == \"message_delta\":\n stop_reason = delta.get(\"stop_reason\") or stop_reason\n if stop_reason:\n received_terminal = True\n break\n if (\n not received_terminal\n and (\n deadline_expired.is_set()\n or (\n deadline_timeout is not None\n and time.monotonic() - started >= deadline_timeout\n )\n )\n ):\n raise RuntimeError(\"OpenCode Go API が時間上限に達しました\")\n elapsed = time.monotonic() - started\n if elapsed >= next_progress:\n _log(\n f\"OpenCode Go ({model_id}) 生成中 \"\n f\"({elapsed:.0f}s / 本文{text_chars}字)\"\n )\n next_progress += 60.0\n finally:\n if fallback_reader is not None:\n fallback_reader.cancel()\n if deadline_expired.is_set() and not received_terminal:\n raise RuntimeError(\"OpenCode Go API が時間上限に達しました\")\n except urllib.error.HTTPError as exc:\n detail = exc.read().decode(\"utf-8\", errors=\"replace\")[:500]\n raise RuntimeError(f\"OpenCode Go API failed (HTTP {exc.code}): {detail}\") from exc\n except (TimeoutError, OSError, ValueError) as exc:\n if deadline_expired.is_set() and not received_terminal:\n raise RuntimeError(\"OpenCode Go API が時間上限に達しました\") from exc\n raise\n text = \"\".join(text_parts)\n if stop_reason == \"max_tokens\":\n raise RuntimeError(\n f\"OpenCode Go API が max_tokens={config.OPENCODE_GO_MAX_TOKENS} に達しました\"\n )\n if not text.strip():\n raise RuntimeError(f\"OpenCode Go API が空の本文を返しました (stop_reason={stop_reason or 'unknown'})\")\n _log(\n f\"OpenCode Go ({model_id}) 完了 \"\n f\"({time.monotonic() - started:.1f}s / 本文{len(text)}字)\"\n )\n return text\n\n\ndef _run_opencode(\n prompt: str,\n model: str,\n agent: str,\n timeout: int | float | None | object = _DEFAULT_WRITE_TIMEOUT,\n) -> str:\n if not model and not agent:\n raise RuntimeError(\n \"OPENCODE_MODEL か OPENCODE_AGENT のどちらかを設定してください (TEXT_BACKEND=opencode)\"\n )\n # 呼び出し元がカスタムエージェントを明示していればそれを優先。未指定（既定）の場合は\n # 下で opencode.json に定義する最小エージェント doci-write を使う。\n # -m（モデル指定）と --agent（エージェント指定）は併用可能な CLI 仕様のため、\n # model の有無に関わらず必ず --agent を付ける。\n effective_agent = agent or \"doci-write\"\n cmd = [\"opencode\", \"run\"]\n if model:\n cmd += [\"-m\", model]\n cmd += [\"--agent\", effective_agent]\n # opencode はエージェント動作でカレントにファイルを書くことがあるため、\n # 使い捨ての作業ディレクトリに隔離する（生成物の repo 汚染を防ぐ）。\n scratch = config.OUTPUT / \".opencode_scratch\"\n scratch.mkdir(parents=True, exist_ok=True)\n # ヘッドレス(無人)実行では build エージェントの \"ask\" 権限(doom_loop /\n # external_directory 等)が応答待ちでブロックし、タイムアウトまでハングすることがある。\n # この使い捨て scratch にスコープ限定の設定を置き、権限を all-allow にして\n # 権限プロンプトで止まらないようにする（ユーザーのグローバル opencode 設定は変更しない）。\n #\n # 加えて doci-write という最小エージェントを定義する。既定の build エージェントは\n # コーディングアシスタント用のフルシステムプロンプト（ツール定義・スキル一覧・\n # グローバル CLAUDE.md 等、約8-10KB）を毎回上乗せするが、本用途は「JSON1個を\n # テキスト生成させるだけ」でありファイル編集・bash実行等のツール能力は不要。\n # prompt は空文字だと既定プロンプトへフォールバックする恐れがあるため半角スペース1つにする。\n (scratch / \"opencode.json\").write_text(\n json.dumps(\n {\n \"$schema\": \"https://opencode.ai/config.json\",\n \"permission\": {\"*\": \"allow\"},\n \"agent\": {\n \"doci-write\": {\n \"mode\": \"primary\",\n \"prompt\": \" \",\n \"tools\": {\"*\": False},\n }\n },\n }\n ),\n encoding=\"utf-8\",\n )\n cmd += [\"--print-logs\", \"--log-level\", \"ERROR\", \"--dir\", str(scratch), prompt]\n proc = subprocess.run(cmd, capture_output=True, text=True, timeout=_write_timeout(timeout))\n if proc.returncode != 0:\n raise RuntimeError(f\"opencode failed (rc={proc.returncode}): {proc.stderr[:500]}\")\n return proc.stdout\n\n\ndef _opencode_cli_model(model: str) -> str:\n \"\"\"OpenCode CLIのagent-only設定を補助段でも維持する。\"\"\"\n candidate = config.OPENCODE_MODEL or (\"\" if config.OPENCODE_AGENT else model)\n return _validate_opencode_cli_model(candidate)\n\n\ndef _validate_opencode_cli_model(model: str) -> str:\n \"\"\"OpenCode CLIへClaudeプロバイダを暗黙に渡さない。\"\"\"\n if model.startswith((\"claude-\", \"anthropic/\", \"opencode-go/claude-\")):\n raise RuntimeError(\n \"TEXT_BACKEND=opencode ではClaudeモデルを使えません。\"\n \" OPENCODE_MODELまたは段別モデルをOpenCode対応値へ変更してください\"\n )\n return model\n\n\ndef _opencode_cli_aux_model(model: str, *, explicit: bool) -> str:\n \"\"\"補助段は段別モデルを優先し、未指定時だけ既存CLI設定を使う。\"\"\"\n if explicit:\n return _validate_opencode_cli_model(model)\n return _opencode_cli_model(model)\n\n\ndef _dispatch(prompt: str, timeout: int | float | None = None) -> str:\n backend = config.TEXT_BACKEND\n model = config.TEXT_MODEL\n if backend == \"claude_cli\":\n if timeout is None:\n return _run_claude_cli(prompt, model)\n return _run_claude_cli(prompt, model, timeout=timeout)\n if backend == \"anthropic\":\n if timeout is None:\n return _run_anthropic(prompt, model)\n return _run_anthropic(prompt, model, timeout=timeout)\n if backend == \"opencode_go\":\n if timeout is None:\n return _run_opencode_go(prompt, config.OPENCODE_MODEL or model)\n return _run_opencode_go(prompt, config.OPENCODE_MODEL or model, timeout=timeout)\n if backend == \"opencode\":\n # agent-only の既存設定では TEXT_MODEL の既定値を混ぜず、\n # _run_opencode 側に空モデルを渡して --agent を有効にする。\n opencode_model = _opencode_cli_model(model)\n if timeout is None:\n return _run_opencode(prompt, opencode_model, config.OPENCODE_AGENT)\n return _run_opencode(\n prompt, opencode_model, config.OPENCODE_AGENT, timeout=timeout\n )\n raise ValueError(f\"unknown TEXT_BACKEND: {backend}\")\n\n\n_SEMANTIC_DUPLICATE_PROMPT = \"\"\"\\\nあなたはショート動画企画の重複審査担当です。「新しい題材候補」が「直近の題材一覧」のいずれかと、\n表現・比喩・切り口を変えただけで結論や主張構造が実質同じ使い回しでないかを判定してください。\n（例: 「見えざる手」と「見えない手」、「成長という名の列車」と「成長という名の神様」は同じ主張の言い換えで重複）\nテーマ領域が同じでも、具体的な結論・視点・題材が明確に異なるものは重複ではありません。\n\n新しい題材候補: {candidate}\n\n直近の題材一覧:\n{numbered}\n\n出力は有効なJSONオブジェクトのみ（説明・コードフェンス禁止）:\n{{\"duplicate\": true または false, \"matched_index\": 一致した番号（1始まり。無ければnull）, \"confidence\": 0から1の確信度, \"reason\": \"判定理由（1文）\"}}\n\"\"\"\n\n\ndef check_semantic_duplicate(\n candidate_topic: str,\n recent_topics: list[str],\n *,\n limit: int = 24,\n text_limit: int = 120,\n) -> tuple[str, float] | None:\n \"\"\"語彙が一致しない言い換え重複をLLMに判定させる。\n\n 一致ありなら (一致した過去題材, 確信度) を返す。通信・応答不良時は\n 誤スキップより見逃しを優先し None を返す（生成を止めない）。\n \"\"\"\n candidate_topic = candidate_topic.strip()\n candidates = [t.strip()[:text_limit] for t in recent_topics if t.strip()][-limit:]\n if not candidate_topic or not candidates:\n return None\n numbered = \"\\n\".join(f\"{i + 1}. {t}\" for i, t in enumerate(candidates))\n prompt = _SEMANTIC_DUPLICATE_PROMPT.format(\n candidate=candidate_topic[:text_limit], numbered=numbered\n )\n try:\n data = _extract_json(_dispatch(prompt, timeout=60))\n except (\n ValueError,\n TimeoutError,\n subprocess.TimeoutExpired,\n RuntimeError,\n OSError,\n ):\n return None\n if not isinstance(data, dict) or not data.get(\"duplicate\"):\n return None\n idx = data.get(\"matched_index\")\n matched = (\n candidates[idx - 1]\n if isinstance(idx, int)\n and not isinstance(idx, bool)\n and 1 <= idx <= len(candidates)\n else candidates[0]\n )\n confidence = data.get(\"confidence\")\n score = confidence if isinstance(confidence, (int, float)) and not isinstance(confidence, bool) else 1.0\n return matched, max(0.0, min(1.0, float(score)))\n\n\n_TITLE_PATTERN_DUPLICATE_PROMPT = \"\"\"\\\nあなたはYouTube動画タイトルの重複審査担当です。「新しいタイトル案」が「直近のタイトル一覧」の\nいずれかと、次の観点で同じ型の使い回しに見えないかを判定してください。\n\n- proper_noun: 固有名詞の重複（同じ企業名・人名・製品名などを核にしている）\n- problem_word: 問題語・キーワードの重複（同じ課題語「改善」「クリック率」等を核にしている）\n- rhetorical_template: 修辞テンプレートの重複（疑問形、「〜するな」型の警告・逆説、\n 「真実」「設計図」「罠」等の紋切り型比喩、「が殺す/壊す」型の煽り構文など、言い回しの骨格が同じ）\n\n上記3観点のうち2つ以上が同じ直近タイトルと重なる場合だけ重複と判定してください。\n1つだけ重なる（例えばどちらも疑問形なだけ）場合は重複ではありません。\n題材・具体的な結論が明確に異なる場合も重複ではありません。\n\n新しいタイトル案: {candidate}\n\n直近のタイトル一覧:\n{numbered}\n\n出力は有効なJSONオブジェクトのみ（説明・コードフェンス禁止）:\n{{\"duplicate\": true または false, \"matched_index\": 一致した番号（1始まり。無ければnull）,\n \"overlapping_axes\": [\"proper_noun\",\"problem_word\",\"rhetorical_template\"]のうち該当するものの配列,\n \"confidence\": 0から1の確信度, \"reason\": \"判定理由（1文）\"}}\n\"\"\"\n\n\ndef check_title_pattern_duplicate(\n candidate_title: str,\n recent_titles: list[str],\n *,\n limit: int = 24,\n text_limit: int = 200,\n) -> dict | None:\n \"\"\"タイトルの修辞パターン重複(固有名詞・問題語・型)をLLMに判定させる。\n\n issue #37: 題材レベルのcooldown一致だけでは、題材(具体的な事実・数値)が\n 違ってもタイトルの型（固有名詞+問題語+疑問形/煽り構文）が繰り返される\n 使い回しを検出できない。一致ありなら判定結果の辞書を返す。通信・応答\n 不良時は誤検出より見逃しを優先しNoneを返す（生成を止めない）。\n \"\"\"\n candidate_title = candidate_title.strip()\n candidates = [t.strip()[:text_limit] for t in recent_titles if t.strip()][-limit:]\n if not candidate_title or not candidates:\n return None\n numbered = \"\\n\".join(f\"{i + 1}. {t}\" for i, t in enumerate(candidates))\n prompt = _TITLE_PATTERN_DUPLICATE_PROMPT.format(\n candidate=candidate_title[:text_limit], numbered=numbered\n )\n try:\n data = _extract_json(_dispatch(prompt, timeout=60))\n except (\n ValueError,\n TimeoutError,\n subprocess.TimeoutExpired,\n RuntimeError,\n OSError,\n ):\n return None\n if not isinstance(data, dict) or not data.get(\"duplicate\"):\n return None\n idx = data.get(\"matched_index\")\n matched = (\n candidates[idx - 1]\n if isinstance(idx, int)\n and not isinstance(idx, bool)\n and 1 <= idx <= len(candidates)\n else candidates[0]\n )\n confidence = data.get(\"confidence\")\n score = (\n confidence\n if isinstance(confidence, (int, float)) and not isinstance(confidence, bool)\n else 1.0\n )\n axes = data.get(\"overlapping_axes\")\n overlapping_axes = (\n [a for a in axes if isinstance(a, str)] if isinstance(axes, list) else []\n )\n reason = data.get(\"reason\")\n return {\n \"matched_title\": matched,\n \"confidence\": max(0.0, min(1.0, float(score))),\n \"overlapping_axes\": overlapping_axes,\n \"reason\": str(reason)[:300] if isinstance(reason, str) else \"\",\n }\n\n\ndef _validate(script: dict) -> dict:\n for k in REQUIRED_KEYS:\n if k not in script:\n raise ValueError(f\"生成JSONに必須キー '{k}' がありません: {list(script)}\")\n if not isinstance(script[\"scenes\"], list) or not script[\"scenes\"]:\n raise ValueError(\"scenes が空です\")\n for s in script[\"scenes\"]:\n s.setdefault(\"caption\", \"\")\n s.setdefault(\"visual_prompt\", \"\")\n s.setdefault(\"motion\", \"\")\n s.setdefault(\"act\", \"\")\n if isinstance(script[\"tags\"], str):\n script[\"tags\"] = [t.strip() for t in script[\"tags\"].split(\",\") if t.strip()]\n _check_cold_open(script.get(\"narration\", \"\"))\n return script\n\n\n_COLD_OPEN_PATTERNS = (\n \"今日は\", \"面白い話\", \"お話です\", \"突然ですが\",\n \"こんにちは\", \"こんばんは\",\n \"知っていますか\",\n)\n\n\ndef _check_cold_open(narration: str) -> None:\n \"\"\"冒頭が『いきなり本編』ルール(output_rules.md)に違反していないかを検証する。\n 実測で narration 冒頭に禁止フレーズが混入する事故が高頻度(24件中13件=54%)で\n 起きていたため、プロンプト指示だけに頼らずコード側でも検証しリトライへ回す。\"\"\"\n head = (narration or \"\")[:60]\n hit = next((p for p in _COLD_OPEN_PATTERNS if p in head), None)\n if hit:\n raise ValueError(f\"冒頭が「いきなり本編」ルール違反（禁止フレーズ「{hit}」を検出）: {head!r}\")\n\n\n# 図表は本来 scenes 側の要素として置く設計だが、執筆モデルが稀に図表マーカー\n# {\"chart_id\":N,\"caption\":\"…\"} を narration 本文へインラインしてしまう。放置すると\n# (1) TTS が JSON をそのまま読み上げ (2) scene 側に chart_id が無く図表が出ない、の二重事故になる。\n_INLINE_CHART = re.compile(\n r'\\s*\\{\\s*\"chart_id\"\\s*:\\s*(\\d+)\\s*,\\s*\"caption\"\\s*:\\s*\"([^\"]*)\"\\s*\\}'\n)\n\n\ndef _strip_chart_markers(text: str) -> str:\n \"\"\"本文中に紛れた図表マーカー JSON を除去し、余分な改行を整える。\"\"\"\n cleaned = _INLINE_CHART.sub(\"\", text or \"\")\n return re.sub(r\"\\n{3,}\", \"\\n\\n\", cleaned).strip()\n\n\ndef _recover_inline_charts(script: dict) -> int:\n \"\"\"narration にインラインされた図表マーカーを scenes の chart_id へ移し、本文から除去する。\n 戻り値は scene へ移せた図表数。本文順＝シーン順とみなし、マーカーの文字位置から\n 相当する scene を推定して chart_id を載せる（既に scene 側にある番号は重複させない）。\"\"\"\n narration = script.get(\"narration\") or \"\"\n scenes = script.get(\"scenes\") or []\n matches = list(_INLINE_CHART.finditer(narration))\n if not matches:\n return 0\n have = {int(s[\"chart_id\"]) for s in scenes if s.get(\"chart_id\") is not None}\n moved = 0\n if scenes:\n span = max(1, len(narration))\n for m in matches:\n cid = int(m.group(1))\n if cid in have:\n continue # 既に scene 側へ正しく置かれている\n idx = min(len(scenes) - 1, int(m.start() / span * len(scenes)))\n # 推定位置から後方優先で、図表未割り当ての scene を探して載せる。\n order = list(range(idx, len(scenes))) + list(range(idx - 1, -1, -1))\n slot = next((j for j in order if scenes[j].get(\"chart_id\") is None), None)\n if slot is None:\n continue\n scenes[slot][\"chart_id\"] = cid\n if m.group(2) and not scenes[slot].get(\"caption\"):\n scenes[slot][\"caption\"] = m.group(2)\n have.add(cid)\n moved += 1\n script[\"narration\"] = _strip_chart_markers(narration)\n return moved\n\n\ndef _log(msg: str) -> None:\n print(f\"[doci] {msg}\", flush=True)\n\n\ndef generate(\n spec: ChannelSpec,\n corner: CornerSpec,\n day: str,\n past_topics: list[str],\n topic_guard: Callable[[str], None] | None = None,\n performance_decision: dict | None = None,\n) -> dict:\n performance_guidance = str(\n (performance_decision or {}).get(\"guidance\") or \"\"\n )\n # 1) 前段リサーチ（issue #6）: 題材選定＋Web裏取り。失敗してもリサーチ無しで続行。\n research = None\n research_enabled = spec.pipeline_get(\"research\", config.SCRIPT_RESEARCH)\n factcheck_enabled = spec.pipeline_get(\"factcheck\", config.SCRIPT_FACTCHECK)\n if research_enabled:\n from . import research as research_mod\n\n _log(f\"前段リサーチ ({config.RESEARCH_BACKEND}+Web)…\")\n try:\n research = research_mod.web_research(\n corner,\n past_topics,\n spec,\n performance_guidance=performance_guidance,\n )\n if research:\n _log(f\"題材: {research.get('topic', '')} / 裏取り事実 {len(research.get('facts', []))}件\")\n except Exception as e: # noqa: BLE001\n _log(f\"リサーチ失敗→リサーチ無しで続行: {e}\")\n research = None\n if research and topic_guard:\n # リサーチで題材が確定した時点で予約する。構成・執筆・音声・映像より前なので、\n # 重複候補に制作コストを使わず、並行runも同じ題材を選べない。\n topic_guard(str(research.get(\"topic\") or \"\"))\n\n # 1.5) 構成プラン（issue #2）: minimax で起承転結＋図表を設計。失敗してもプラン無しで続行。\n plan = None\n if spec.pipeline_get(\"plan\", config.SCRIPT_PLAN):\n from . import plan as plan_mod\n\n _log(f\"構成プラン (minimax) …\")\n try:\n plan = plan_mod.make_plan(corner, research)\n if plan:\n _log(f\"構成: 起承転結{len(plan.get('beats', []))}ビート / 図表 {len(plan.get('charts', []))}個\")\n except Exception as e: # noqa: BLE001\n _log(f\"プラン失敗→プラン無しで続行: {e}\")\n plan = None\n\n # 2) 執筆（qwen3.7-plus 等）。リサーチの具体＋プランの構成/図表に沿わせる。\n # 稀に不完全JSONを返すため再生成で吸収。\n prompt = corners.build_prompt(\n spec,\n corner,\n day,\n past_topics,\n research=research,\n plan=plan,\n performance_guidance=performance_guidance,\n )\n script = None\n last_err: Exception | None = None\n draft_started = _monotonic()\n draft_total_timeout = (\n config.SCRIPT_DRAFT_TOTAL_TIMEOUT\n if config.SCRIPT_DRAFT_TOTAL_TIMEOUT > 0\n else None\n )\n for attempt in range(1, config.SCRIPT_DRAFT_RETRIES + 1):\n attempt_timeout = None\n if draft_total_timeout is not None:\n remaining_budget = draft_total_timeout - (_monotonic() - draft_started)\n if remaining_budget <= 0:\n detail = \"\"\n if last_err is not None:\n detail = (\n f\" (直前の失敗: {type(last_err).__name__}: \"\n f\"{str(last_err)[:160]})\"\n )\n last_err = TimeoutError(\n f\"執筆段全体の時間上限に達しました{detail}\"\n )\n break\n per_attempt_timeout = _whole_write_timeout()\n attempt_timeout = (\n min(per_attempt_timeout, remaining_budget)\n if per_attempt_timeout is not None\n else remaining_budget\n )\n try:\n script = _validate(_extract_json(_dispatch(prompt, timeout=attempt_timeout)))\n break\n except (ValueError, TimeoutError, subprocess.TimeoutExpired, RuntimeError, OSError) as e:\n # JSON不良/必須キー不足（ValueError）だけでなく、執筆バックエンドのタイムアウト\n # (TimeoutExpired)・異常終了(RuntimeError)・ネットワーク失敗(OSError)も一過性とみなし\n # 再試行する（qwen 等が稀に固まり、1回の失敗で通し全体が即死するのを防ぐ）。\n last_err = e\n # 例外メッセージにプロンプト全文が載ることがあるため要約のみログする。\n _log(\n f\"執筆失敗(試行{attempt}/{config.SCRIPT_DRAFT_RETRIES})→再生成: \"\n f\"{type(e).__name__}: {str(e)[:160]}\"\n )\n if script is None:\n raise RuntimeError(f\"執筆が規定回数で揃いませんでした: {last_err}\")\n if not research and topic_guard:\n # リサーチがフォールバックしたrunでも、動画生成・投稿へ進む前に\n # タイトルと概要の先頭行を題材としてcooldownを適用する。\n topic_guard(\n f\"{script.get('title', '')} \"\n f\"{str(script.get('description') or '').splitlines()[0] if script.get('description') else ''}\"\n )\n\n # 2.4) 救済: 執筆モデルが narration に図表マーカーをインラインした場合、本文から除去して\n # 相当する scene へ chart_id を移す（音声へのJSON混入と図表欠落を同時に防ぐ）。\n recovered = _recover_inline_charts(script)\n if recovered:\n _log(f\"本文混入の図表マーカーを {recovered} 件回収（→scene へ移動）\")\n\n # 2.5) chart_id を実図表仕様に解決（データはプラン＝minimax由来を正とし取り違えを防ぐ）。\n if plan and plan.get(\"charts\"):\n by_id = {c[\"id\"]: c for c in plan[\"charts\"]}\n used = 0\n for s in script[\"scenes\"]:\n cid = s.get(\"chart_id\")\n if cid is not None and int(cid) in by_id:\n s[\"chart\"] = by_id[int(cid)]\n s.pop(\"chart_id\", None)\n used += 1\n if used:\n _log(f\"図表を {used} シーンに配置\")\n\n # 2.9) OpenCode系ファクトチェックだけを有効にした場合は、下書き後に\n # 資料取得だけを行う。research=false の題材選定・cooldown意味は変えない。\n factcheck_research = research\n if (\n factcheck_enabled\n and config.SCRIPT_FACTCHECK_RESEARCH\n and config.FACTCHECK_BACKEND in {\"opencode\", \"opencode_go\"}\n and not research_enabled\n ):\n from . import research as research_mod\n\n _log(f\"ファクトチェック用リサーチ ({config.FACTCHECK_BACKEND}+Web)…\")\n try:\n factcheck_research = research_mod.web_research(\n corner,\n past_topics,\n spec,\n performance_guidance=performance_guidance,\n backend_override=config.FACTCHECK_BACKEND,\n model_override=config.FACTCHECK_MODEL,\n model_explicit_override=config._FACTCHECK_MODEL_EXPLICIT,\n focus_text=(\n f\"{script.get('title', '')}\\n{script.get('narration', '')}\"\n ),\n require_youtube_examples=False,\n )\n except Exception as e: # noqa: BLE001\n _log(f\"ファクトチェック用リサーチ失敗→原文維持: {e}\")\n factcheck_research = None\n\n # 3) 後段ファクトチェック（issue #6）: 別モデル＋Web検証で narration を自動修正。\n if factcheck_enabled:\n from . import factcheck\n\n _log(f\"後段ファクトチェック ({config.FACTCHECK_BACKEND}+Web)…\")\n try:\n fc = factcheck.verify_and_correct(script[\"narration\"], factcheck_research)\n if fc and fc.get(\"narration\", \"\").strip():\n issues = fc.get(\"issues\") or []\n if fc.get(\"changed\") and issues:\n _log(f\"ファクトチェック: {len(issues)}件修正\")\n script[\"narration\"] = _strip_chart_markers(fc[\"narration\"])\n script[\"_factcheck\"] = issues\n except factcheck.FactcheckSourcesUnavailableError:\n _log(\"ファクトチェック資料がないため失敗として扱います\")\n if config.SCRIPT_FACTCHECK_REQUIRE_SOURCES:\n raise\n except Exception as e: # noqa: BLE001\n _log(f\"ファクトチェック失敗→修正なしで続行: {e}\")\n\n script[\"_corner\"] = corner.key\n script[\"_speaker\"] = spec.voice_for(corner).speaker\n script[\"_channel\"] = spec.id\n script[\"_date\"] = day\n if research:\n script[\"_research\"] = research\n if performance_decision:\n script[\"_performance_feedback\"] = performance_decision\n return script\n\n\ndef main() -> None:\n ap = argparse.ArgumentParser(description=\"台本生成 (OpenCode Go / qwen3.7-plus)\")\n ap.add_argument(\"--channel\", help=\"チャンネルID（未指定時は既定チャンネル）\")\n ap.add_argument(\"--corner\", help=\"コーナーID\")\n ap.add_argument(\"--date\", default=_date.today().isoformat())\n args = ap.parse_args()\n spec = channel.load(args.channel or channel.default_channel())\n corner_key = args.corner or spec.rotation[0]\n if corner_key not in spec.corners:\n ap.error(\n f\"unknown corner for channel {spec.id}: {corner_key}; \"\n f\"choose from {', '.join(spec.corners)}\"\n )\n corner = spec.corners[corner_key]\n voice = spec.voice_for(corner)\n script = generate(spec, corner, args.date, past_topics=[])\n print(json.dumps(script, ensure_ascii=False, indent=2))\n print(\n f\"\\n--- channel={spec.id} corner={corner.key} voice={corner.voice_key} \"\n f\"speaker={voice.speaker} \"\n f\"narration_chars={len(script['narration'])} scenes={len(script['scenes'])} ---\",\n file=sys.stderr,\n )\n\n\nif __name__ == \"__main__\":\n main()\n"}}}} |