1 line
17 KiB
JSON
1 line
17 KiB
JSON
{"repo": "christophezhao/hydramind", "files": {"src/hydramind/cli_doctor.py": {"b59831436d01ca3d6a07a8f1ca41c4839bab73bc": {"t": 1785745445.5289073, "n": 16773, "h": "0ce81096a5ae764d", "c": "\"\"\"Doctor command registration and diagnostics for the HydraMind CLI.\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport asyncio\nimport json\nimport os\nfrom collections.abc import Callable\nfrom pathlib import Path\nfrom typing import Any\n\nfrom hydramind.cli_doctor_goal_scenario import run_doctor_goal_scenario\nfrom hydramind.cli_doctor_tools import (\n doctor_tool_args,\n run_doctor_tools,\n)\nfrom hydramind.cli_support import (\n env_present,\n split_values,\n)\nfrom hydramind.control import (\n ControlPlane,\n InMemorySessionStore,\n RuntimeSession,\n SessionService,\n WorkflowBlueprint,\n WorkflowNodeSpec,\n)\nfrom hydramind.harness import (\n Message,\n MessageRole,\n ModelProvider,\n StopReason,\n ToolCall,\n ToolSpec,\n)\nfrom hydramind.observability import (\n Emitter,\n JsonlObserver,\n ListObserver,\n ObservationEvent,\n ObservationEventKind,\n)\nfrom hydramind.orchestration import OrchestratorAgent\nfrom hydramind.runtime import create_provider, load_env_file\nfrom hydramind.testing import MockProvider, ScriptedTurn\nfrom hydramind.tools import (\n ToolContext,\n ToolRegistry,\n build_default_tool_registry,\n default_external_tool_hosts,\n)\n\nProviderFactory = Callable[[str], ModelProvider]\n\n\ndef register_doctor_commands(subparsers: Any) -> None:\n doctor = subparsers.add_parser(\"doctor\", help=\"run provider/tool diagnostics\")\n doctor_sub = doctor.add_subparsers(dest=\"doctor_command\", required=True)\n env = doctor_sub.add_parser(\n \"env\",\n help=\"check required environment keys without printing values\",\n )\n env.add_argument(\"--env-file\", default=\".env\", help=\"dotenv file to load\")\n env.add_argument(\n \"--profile\",\n choices=(\"all\", \"providers\", \"tools\"),\n default=\"all\",\n help=\"environment profile to check\",\n )\n env.add_argument(\n \"--include-missing-template\",\n action=\"store_true\",\n help=\"include empty KEY= template lines for missing variables\",\n )\n providers = doctor_sub.add_parser(\n \"providers\",\n help=\"smoke test configured LLM providers\",\n )\n providers.add_argument(\"--env-file\", default=\".env\", help=\"dotenv file to load\")\n providers.add_argument(\"--provider\", default=\"env\", help=\"provider override\")\n providers.add_argument(\n \"--roles\",\n action=\"append\",\n default=[],\n help=\"role or comma-separated roles to test; default planner\",\n )\n providers.add_argument(\n \"--prompt\",\n default=\"Reply with OK.\",\n help=\"small smoke prompt\",\n )\n providers.add_argument(\n \"--max-tokens\",\n type=int,\n default=32,\n help=\"maximum output tokens for each provider smoke request\",\n )\n providers.add_argument(\n \"--timeout-seconds\",\n type=float,\n default=None,\n help=\"override provider HTTP timeout for diagnostics\",\n )\n tools = doctor_sub.add_parser(\n \"tools\",\n help=\"inspect and execute registered tools\",\n )\n tools.add_argument(\"--env-file\", default=\".env\", help=\"dotenv file to load\")\n tools.add_argument(\n \"--live-tools\",\n action=\"store_true\",\n help=\"allow live tool APIs\",\n )\n tools.add_argument(\n \"--tool\",\n action=\"append\",\n default=[],\n help=\"tool name or comma-separated tool names to execute\",\n )\n tools.add_argument(\n \"--artifact-root\",\n default=\"artifacts/doctor\",\n help=\"artifact root\",\n )\n tool_loop = doctor_sub.add_parser(\n \"tool-loop\",\n help=\"smoke test provider-driven tool-call loop with trace evidence\",\n )\n tool_loop.add_argument(\"--env-file\", default=\".env\", help=\"dotenv file to load\")\n tool_loop.add_argument(\"--provider\", default=\"env\", help=\"provider override\")\n tool_loop.add_argument(\"--tool\", default=\"time.now\", help=\"single tool to expose\")\n tool_loop.add_argument(\n \"--artifact-root\",\n default=\"artifacts/doctor-tool-loop\",\n help=\"artifact root\",\n )\n tool_loop.add_argument(\"--trace-path\", default=None, help=\"JSONL trace path\")\n tool_loop.add_argument(\n \"--live-tools\",\n action=\"store_true\",\n help=\"allow live tool APIs\",\n )\n tool_loop.add_argument(\n \"--timeout-seconds\",\n type=float,\n default=None,\n help=\"overall smoke timeout in seconds\",\n )\n tool_loop.add_argument(\n \"--prompt\",\n default=(\n \"Call the exposed tool exactly once, wait for its result, then reply \"\n \"with a compact JSON object.\"\n ),\n help=\"instruction sent to the provider\",\n )\n goal_scenario = doctor_sub.add_parser(\n \"goal-scenario\",\n help=\"run a goal-driven scenario and validate required tool evidence\",\n )\n goal_scenario.add_argument(\n \"--env-file\",\n default=\".env\",\n help=\"dotenv file to load\",\n )\n goal_scenario.add_argument(\"--provider\", default=\"env\", help=\"provider override\")\n goal_scenario.add_argument(\n \"--planner\",\n default=\"auto\",\n choices=(\"auto\", \"model\"),\n help=\"goal planner implementation\",\n )\n goal_scenario.add_argument(\n \"--objective\",\n default=None,\n help=\"scenario objective; defaults to a required-tool exercise\",\n )\n goal_scenario.add_argument(\n \"--tool\",\n action=\"append\",\n default=[],\n help=\"tool name or comma-separated tool names available to the goal\",\n )\n goal_scenario.add_argument(\n \"--required-tool\",\n action=\"append\",\n default=[],\n help=\"tool name or comma-separated tool names that must succeed\",\n )\n goal_scenario.add_argument(\n \"--artifact-root\",\n default=\"artifacts/doctor-goal-scenario\",\n help=\"artifact root\",\n )\n goal_scenario.add_argument(\n \"--trace-path\",\n default=None,\n help=\"JSONL trace path\",\n )\n goal_scenario.add_argument(\n \"--live-tools\",\n action=\"store_true\",\n help=\"allow live tool APIs\",\n )\n goal_scenario.add_argument(\n \"--timeout-seconds\",\n type=float,\n default=None,\n help=\"overall scenario timeout in seconds\",\n )\n\n\ndef run_doctor(\n args: argparse.Namespace,\n *,\n provider_factory: ProviderFactory = create_provider,\n) -> int:\n if args.doctor_command == \"env\":\n return _doctor_env(args)\n if args.doctor_command == \"providers\":\n return asyncio.run(_doctor_providers(args, provider_factory))\n if args.doctor_command == \"tools\":\n return asyncio.run(run_doctor_tools(args))\n if args.doctor_command == \"tool-loop\":\n return asyncio.run(_doctor_tool_loop(args, provider_factory))\n if args.doctor_command == \"goal-scenario\":\n return asyncio.run(run_doctor_goal_scenario(args))\n return 2\n\n\ndef _doctor_env(args: argparse.Namespace) -> int:\n load_env_file(args.env_file)\n tool_registry = build_default_tool_registry(context=ToolContext(dry_run=True))\n tool_keys = sorted(\n {\n key\n for keys in tool_registry.env_requirements().values()\n for key in keys\n }\n )\n profiles: dict[str, list[str]] = {\n \"providers\": [\"DEEPSEEK_API_KEY\", \"KIMI_API_KEY\", \"GLM_API_KEY\"],\n \"tools\": tool_keys,\n }\n selected = (\n profiles if args.profile == \"all\" else {args.profile: profiles[args.profile]}\n )\n groups: dict[str, dict[str, Any]] = {}\n missing_template: list[str] = []\n for profile, keys in selected.items():\n checks = [{\"key\": key, \"present\": env_present(key)} for key in keys]\n missing_template.extend(\n f\"{item['key']}=\" for item in checks if not item[\"present\"]\n )\n groups[profile] = {\n \"ok\": all(item[\"present\"] for item in checks),\n \"keys\": checks,\n }\n payload: dict[str, Any] = {\n \"env_file\": str(args.env_file),\n \"ok\": all(group[\"ok\"] for group in groups.values()),\n \"profiles\": groups,\n }\n if args.include_missing_template:\n payload[\"missing_template\"] = missing_template\n print(json.dumps(payload, ensure_ascii=False, default=str))\n return 0 if payload[\"ok\"] else 1\n\n\nasync def _doctor_providers(\n args: argparse.Namespace,\n provider_factory: ProviderFactory,\n) -> int:\n load_env_file(args.env_file)\n if args.timeout_seconds is not None:\n for key in (\n \"DEEPSEEK_TIMEOUT_SECONDS\",\n \"KIMI_TIMEOUT_SECONDS\",\n \"GLM_TIMEOUT_SECONDS\",\n ):\n os.environ[key] = str(args.timeout_seconds)\n provider = provider_factory(args.provider)\n roles = split_values(args.roles) or [\"planner\"]\n results: list[dict[str, Any]] = []\n for role in roles:\n try:\n invocation = provider.complete(\n messages=[Message(role=MessageRole.USER, content=args.prompt)],\n role=role,\n max_tokens=args.max_tokens,\n )\n result = (\n await asyncio.wait_for(invocation, timeout=args.timeout_seconds)\n if args.timeout_seconds is not None\n else await invocation\n )\n results.append(\n {\n \"role\": role,\n \"ok\": True,\n \"provider\": provider.name,\n \"model_id\": result.model_id,\n \"stop_reason\": result.stop_reason.value,\n \"content_preview\": result.content[:80],\n }\n )\n except Exception as exc:\n results.append(\n {\n \"role\": role,\n \"ok\": False,\n \"provider\": provider.name,\n \"error_type\": type(exc).__name__,\n \"error\": str(exc),\n }\n )\n close = getattr(provider, \"close\", None)\n if callable(close):\n await close()\n print(json.dumps({\"providers\": results}, ensure_ascii=False, default=str))\n return 0 if all(item[\"ok\"] for item in results) else 1\n\n\nasync def _doctor_tool_loop(\n args: argparse.Namespace,\n provider_factory: ProviderFactory,\n) -> int:\n provider = None\n trace_path = (\n Path(args.trace_path)\n if args.trace_path\n else Path(args.artifact_root) / \"tool-loop-trace.jsonl\"\n )\n try:\n load_env_file(args.env_file)\n artifact_root = Path(args.artifact_root)\n if trace_path.exists():\n trace_path.unlink()\n registry = build_default_tool_registry(\n context=ToolContext(\n artifact_root=artifact_root,\n dry_run=not bool(args.live_tools),\n network_access=bool(args.live_tools),\n allowed_network_hosts=default_external_tool_hosts(dict(os.environ)),\n tool_timeout_seconds=120.0,\n )\n )\n tool_name = str(args.tool).strip()\n if not tool_name or tool_name not in registry.names():\n payload = {\n \"ok\": False,\n \"reason\": \"unknown_tool\",\n \"tool\": tool_name,\n \"known_tools\": registry.names(),\n \"trace_path\": str(trace_path),\n }\n print(json.dumps(payload, ensure_ascii=False, default=str))\n return 1\n provider = _create_doctor_tool_loop_provider(\n args.provider,\n tool_name,\n provider_factory,\n )\n observer = ListObserver()\n emitter = Emitter([JsonlObserver(trace_path), observer])\n service = SessionService(InMemorySessionStore(), emitter=emitter)\n control = ControlPlane(service)\n agent = OrchestratorAgent(\n provider=provider,\n control=control,\n workflow=WorkflowBlueprint(\n name=\"doctor_tool_loop\",\n nodes=(\n WorkflowNodeSpec(\n key=\"tool_loop\",\n role=\"executor\",\n description=f\"Call {tool_name} and finalize.\",\n ),\n ),\n ),\n tool_provider=_SelectedToolProvider(registry, (tool_name,)),\n tool_runner=registry,\n emitter=emitter,\n )\n run = _run_doctor_tool_loop(\n agent,\n service,\n instruction=str(args.prompt),\n tool_name=tool_name,\n )\n session = (\n await asyncio.wait_for(run, timeout=args.timeout_seconds)\n if args.timeout_seconds is not None\n else await run\n )\n evidence = _tool_loop_evidence(\n events=observer.events,\n tool_name=tool_name,\n session_status=session.status.value,\n trace_path=trace_path,\n provider_name=provider.name,\n )\n print(json.dumps(evidence, ensure_ascii=False, default=str))\n return 0 if evidence[\"ok\"] else 1\n except Exception as exc:\n payload = {\n \"ok\": False,\n \"reason\": \"exception\",\n \"provider\": getattr(provider, \"name\", str(args.provider)),\n \"tool\": str(args.tool),\n \"trace_path\": str(trace_path),\n \"error_type\": type(exc).__name__,\n \"error\": str(exc),\n }\n print(json.dumps(payload, ensure_ascii=False, default=str))\n return 1\n finally:\n if provider is not None:\n close = getattr(provider, \"close\", None)\n if callable(close):\n await close()\n\n\nclass _SelectedToolProvider:\n def __init__(self, registry: ToolRegistry, names: tuple[str, ...]) -> None:\n self._registry = registry\n self._names = names\n\n def tools_for(self, node_key: str) -> list[ToolSpec]:\n del node_key\n return self._registry.specs_for(self._names)\n\n\ndef _create_doctor_tool_loop_provider(\n provider_name: str,\n tool_name: str,\n provider_factory: ProviderFactory,\n) -> ModelProvider:\n normalized = provider_name.lower().replace(\"-\", \"_\")\n if normalized == \"mock\":\n return MockProvider(\n scripted=[\n ScriptedTurn(\n content=\"\",\n tool_calls=(\n ToolCall(\n id=f\"doctor-{tool_name}\",\n name=tool_name,\n arguments=doctor_tool_args(tool_name),\n ),\n ),\n stop_reason=StopReason.TOOL_USE,\n ),\n ScriptedTurn(content='{\"doctor_tool_loop\": true}'),\n ]\n )\n return provider_factory(provider_name)\n\n\nasync def _run_doctor_tool_loop(\n agent: OrchestratorAgent,\n service: SessionService,\n *,\n instruction: str,\n tool_name: str,\n) -> RuntimeSession:\n session = await agent.start_session(\n input_payload={\"instruction\": instruction, \"required_tool\": tool_name}\n )\n await agent.run_session(session.id)\n return await service.get_session(session.id)\n\n\ndef _tool_loop_evidence(\n *,\n events: list[ObservationEvent],\n tool_name: str,\n session_status: str,\n trace_path: Path,\n provider_name: str,\n) -> dict[str, Any]:\n event_counts: dict[str, int] = {}\n for event in events:\n event_counts[event.kind.value] = event_counts.get(event.kind.value, 0) + 1\n tool_started = [\n event\n for event in events\n if event.kind is ObservationEventKind.TOOL_CALL_STARTED\n and event.detail.get(\"tool_name\") == tool_name\n ]\n tool_completed = [\n event\n for event in events\n if event.kind is ObservationEventKind.TOOL_CALL_COMPLETED\n ]\n model_completed = [\n event\n for event in events\n if event.kind is ObservationEventKind.MODEL_INVOKE_COMPLETED\n ]\n\n reason = \"passed\"\n if session_status != \"completed\":\n reason = \"session_not_completed\"\n elif not trace_path.exists():\n reason = \"trace_missing\"\n elif not tool_started:\n reason = \"no_tool_call\"\n elif not tool_completed:\n reason = \"tool_result_missing\"\n elif len(model_completed) < 2:\n reason = \"final_model_missing\"\n return {\n \"ok\": reason == \"passed\",\n \"reason\": reason,\n \"provider\": provider_name,\n \"tool\": tool_name,\n \"session_status\": session_status,\n \"trace_path\": str(trace_path),\n \"event_counts\": dict(sorted(event_counts.items())),\n \"model_invoke_completed\": len(model_completed),\n \"tool_call_started\": len(tool_started),\n \"tool_call_completed\": len(tool_completed),\n }\n"}}}} |