1 line
21 KiB
JSON
1 line
21 KiB
JSON
{"repo": "bebemdjd/valtune-agent", "files": {"cli.py": {"49fcd043526567db7675821cc7b173f84d1b535f": {"t": 1785674059.3962252, "n": 9771, "h": "448081e4478a4796", "c": "\"\"\"\r\nValTune-Agent CLI: Main entry point.\r\n\r\nUsage:\r\n python cli.py --base-config configs/base.yaml --train-cmd \"python train.py --cfg {config} --run_dir {run_dir}\"\r\n\r\nSetup:\r\n export DEEPSEEK_API_KEY=\"your-api-key\"\r\n python cli.py --base-config configs/base.yaml --train-cmd \"...\"\r\n\"\"\"\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nfrom pathlib import Path\r\nfrom typing import Optional\r\n\r\nfrom dotenv import load_dotenv\r\nfrom langchain_deepseek import ChatDeepSeek\r\nfrom langchain_core.tools import tool\r\n\r\nfrom agents.schemas import TuneDecision, MetricsSnapshot, ConfigPatch\r\nfrom agents.tuner_prompt import SYSTEM_PROMPT, TUNING_PROMPT_TEMPLATE, CONSTRAINTS, format_history\r\nfrom agents.graph import TuningGraph\r\nfrom tools.config_io import ConfigManager\r\nfrom tools.metrics import MetricsParser\r\nfrom tools.runner import TrainingRunner\r\nfrom tools.hardware import HardwareAnalyzer\r\n\r\n\r\n# Load environment variables (DEEPSEEK_API_KEY, etc.)\r\nload_dotenv()\r\n\r\n\r\ndef setup_llm(api_key_override=None):\r\n \"\"\"Initialize ChatDeepSeek with structured output.\"\"\"\r\n # Priority: command line argument > environment variable\r\n api_key = api_key_override or os.getenv(\"DEEPSEEK_API_KEY\")\r\n if not api_key:\r\n print(\"\\n⚠️ Error: DEEPSEEK_API_KEY not set\")\r\n print(\"Please set using one of the following methods:\")\r\n print(\" 1. Command line: --api-key 'your-key'\")\r\n print(\" 2. Environment: $env:DEEPSEEK_API_KEY='your-key' (PowerShell)\")\r\n print(\" 3. Environment: export DEEPSEEK_API_KEY='your-key' (Bash)\")\r\n print(\"\\n或使用 --mock-llm 参数进行测试（无需 API）\")\r\n raise ValueError(\"DEEPSEEK_API_KEY not set in environment\")\r\n \r\n llm = ChatDeepSeek(\r\n model=\"deepseek-chat\",\r\n api_key=api_key,\r\n temperature=0.7,\r\n max_tokens=1000,\r\n )\r\n \r\n # Enable structured output with Pydantic schema\r\n # 输出严格对准 TuneDecision\r\n llm_with_schema = llm.with_structured_output(\r\n TuneDecision,\r\n method=\"json_schema\",\r\n strict=True\r\n )\r\n \r\n return llm_with_schema\r\n\r\n\r\n\r\n\r\ndef main():\r\n parser = argparse.ArgumentParser(description=\"ValTune-Agent CLI\")\r\n parser.add_argument(\r\n \"--base-config\",\r\n type=str,\r\n default=\"configs/base.yaml\",\r\n help=\"Path to base.yaml\"\r\n )\r\n parser.add_argument(\r\n \"--train-cmd\",\r\n type=str,\r\n required=True,\r\n help='Training command template, e.g., \"python train.py --cfg {config} --run_dir {run_dir}\"'\r\n )\r\n parser.add_argument(\r\n \"--max-rounds\",\r\n type=int,\r\n default=20,\r\n help=\"Maximum training rounds\"\r\n )\r\n parser.add_argument(\r\n \"--timeout\",\r\n type=int,\r\n default=None,\r\n help=\"Timeout per training round (seconds)\"\r\n )\r\n parser.add_argument(\r\n \"--simple-loop\",\r\n action=\"store_true\",\r\n help=\"Use simple loop instead of LangGraph workflow\"\r\n )\r\n parser.add_argument(\r\n \"--api-key\",\r\n type=str,\r\n default=\"sk-\",\r\n help=\"DeepSeek API key (overrides environment variable)\"\r\n )\r\n \r\n args = parser.parse_args()\r\n \r\n # Initialize components\r\n config_manager = ConfigManager(args.base_config)\r\n metrics_parser = MetricsParser()\r\n runner = TrainingRunner(args.train_cmd, config_manager)\r\n llm = setup_llm(api_key_override=args.api_key)\r\n \r\n # Load base config\r\n base_config = config_manager.load_yaml(args.base_config)\r\n \r\n print(f\"Starting ValTune-Agent loop (max {args.max_rounds} rounds)\")\r\n print(f\"Base config: {args.base_config}\")\r\n print(f\"Training command: {args.train_cmd}\")\r\n print(f\"Mode: {'Simple Loop' if args.simple_loop else 'LangGraph (Default)'}\")\r\n print(f\"LLM: DeepSeek API\\n\")\r\n \r\n # Default to LangGraph workflow unless --simple-loop is specified\r\n if not args.simple_loop:\r\n # Use LangGraph workflow (default mode)\r\n print(\"Using LangGraph workflow (default)...\\n\")\r\n \r\n # Initialize LangGraph with all components\r\n tuning_graph = TuningGraph(\r\n llm_client=llm,\r\n config_manager=config_manager,\r\n runner=runner,\r\n metrics_parser=metrics_parser\r\n )\r\n \r\n # Prepare initial state for workflow\r\n initial_state = {\r\n \"base_config_path\": args.base_config,\r\n \"current_config\": base_config,\r\n \"round_number\": 0,\r\n \"metrics_history\": [],\r\n \"decisions\": [],\r\n \"should_stop\": False,\r\n \"max_rounds\": args.max_rounds\r\n }\r\n \r\n # Execute the LangGraph workflow\r\n final_state = tuning_graph.run(initial_state)\r\n \r\n # Print final summary\r\n print(f\"\\n{'=' * 60}\")\r\n print(\"Tuning Loop Completed (LangGraph)\")\r\n print(f\"{'=' * 60}\")\r\n print(f\"Total rounds: {len(final_state.get('decisions', []))}\")\r\n metrics_history = final_state.get('metrics_history', [])\r\n if metrics_history:\r\n print(f\"Best mAP50: {max([m.get('map_50', 0) for m in metrics_history if m.get('map_50')], default=0):.4f}\")\r\n print(f\"Best mAP50-95: {max([m.get('map_50_95', 0) for m in metrics_history if m.get('map_50_95')], default=0):.4f}\")\r\n \r\n else:\r\n # Use simple sequential loop mode\r\n print(\"Using simple loop...\\n\")\r\n \r\n # Track state across rounds: metrics and decisions\r\n state = {\r\n \"round_number\": 0,\r\n \"base_config\": base_config,\r\n \"metrics_history\": [],\r\n \"decisions\": [],\r\n }\r\n \r\n for round_num in range(args.max_rounds):\r\n print(f\"\\n{'=' * 60}\")\r\n print(f\"Round {round_num}\")\r\n print(f\"{'=' * 60}\")\r\n \r\n state[\"round_number\"] = round_num\r\n \r\n # 1. Launch training with current configuration\r\n config = state[\"base_config\"]\r\n print(f\"Launching training with config:\")\r\n for k, v in config.items():\r\n print(f\" {k}: {v}\")\r\n \r\n run_dir = runner.launch_training(config, round_num, timeout=args.timeout)\r\n \r\n # 2. Parse metrics from training output\r\n metrics = metrics_parser.parse_metrics(str(run_dir))\r\n state[\"metrics_history\"].append(metrics)\r\n print(f\"\\nValidation Metrics:\")\r\n print(f\" mAP50: {metrics.map_50:.4f}\" if metrics.map_50 else \" mAP50: N/A\")\r\n print(f\" mAP50-95: {metrics.map_50_95:.4f}\" if metrics.map_50_95 else \" mAP50-95: N/A\")\r\n print(f\" Precision: {metrics.precision:.4f}\" if metrics.precision else \" Precision: N/A\")\r\n print(f\" Recall: {metrics.recall:.4f}\" if metrics.recall else \" Recall: N/A\")\r\n print(f\" Val Loss: {metrics.val_loss:.4f}\" if metrics.val_loss else \" Val Loss: N/A\")\r\n \r\n # 3. Query LLM for optimization decision\r\n history_text = format_history(state[\"decisions\"], state[\"metrics_history\"])\r\n \r\n prompt = TUNING_PROMPT_TEMPLATE.format(\r\n round_number=round_num,\r\n history=history_text,\r\n map_50=metrics.map_50 or 0.0,\r\n map_50_95=metrics.map_50_95 or 0.0,\r\n precision=metrics.precision or 0.0,\r\n recall=metrics.recall or 0.0,\r\n val_loss=metrics.val_loss or 0.0,\r\n epoch=metrics.epoch,\r\n current_config=json.dumps(config, indent=2)\r\n )\r\n \r\n print(f\"\\nQuerying LLM for optimization suggestions...\")\r\n \r\n try:\r\n decision = llm.invoke(SYSTEM_PROMPT + \"\\n\\n\" + prompt)\r\n # Ensure decision is a TuneDecision object\r\n if not isinstance(decision, TuneDecision):\r\n decision = TuneDecision(**decision)\r\n state[\"decisions\"].append(decision)\r\n except Exception as e:\r\n print(f\"Error querying LLM: {e}\")\r\n break\r\n \r\n print(f\"\\nOptimization Decision:\")\r\n print(f\" Action: {decision.action}\")\r\n print(f\" Reason: {decision.reason}\")\r\n if decision.patch:\r\n print(f\" Suggested Changes: {decision.patch.dict(exclude_none=True)}\")\r\n \r\n # 4. Check if LLM recommends stopping\r\n if decision.action == \"stop\":\r\n print(f\"\\nConvergence detected - stopping tuning.\")\r\n break\r\n \r\n # 5. Apply suggested changes for next round\r\n if decision.patch:\r\n state[\"base_config\"] = config_manager.apply_patch(\r\n state[\"base_config\"],\r\n decision.patch\r\n )\r\n print(f\"\\nConfiguration updated for next round\")\r\n \r\n # Record this round's metadata\r\n runner.save_round_metadata(\r\n round_num,\r\n config,\r\n metrics.dict(),\r\n decision.dict() if hasattr(decision, 'dict') else decision\r\n )\r\n \r\n # Print final optimization summary\r\n print(f\"\\n{'=' * 60}\")\r\n print(\"Hyperparameter Tuning Complete\")\r\n print(f\"{'=' * 60}\")\r\n print(f\"Total optimization rounds: {len(state['decisions'])}\")\r\n print(f\"Best mAP50: {max([m.map_50 for m in state['metrics_history'] if m.map_50], default=0):.4f}\")\r\n print(f\"Best mAP50-95: {max([m.map_50_95 for m in state['metrics_history'] if m.map_50_95], default=0):.4f}\")\r\n\r\n\r\nif __name__ == \"__main__\":\r\n main()\r\n\r\n"}, "efab4d5442345e3dad3ae9af77c4fef371d08eb6": {"t": 1785752924.563075, "n": 9801, "h": "017e7878fc05bf13", "c": "\"\"\"\r\nValTune-Agent CLI: Main entry point.\r\n\r\nUsage:\r\n python cli.py --base-config configs/base.yaml --train-cmd \"python train.py --cfg {config} --run_dir {run_dir}\"\r\n\r\nSetup:\r\n export DEEPSEEK_API_KEY=\"your-api-key\"\r\n python cli.py --base-config configs/base.yaml --train-cmd \"...\"\r\n\"\"\"\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nfrom pathlib import Path\r\nfrom typing import Optional\r\n\r\nfrom dotenv import load_dotenv\r\nfrom langchain_deepseek import ChatDeepSeek\r\nfrom langchain_core.tools import tool\r\n\r\nfrom agents.schemas import TuneDecision, MetricsSnapshot, ConfigPatch\r\nfrom agents.tuner_prompt import SYSTEM_PROMPT, TUNING_PROMPT_TEMPLATE, CONSTRAINTS, format_history\r\nfrom agents.graph import TuningGraph\r\nfrom tools.config_io import ConfigManager\r\nfrom tools.metrics import MetricsParser\r\nfrom tools.runner import TrainingRunner\r\nfrom tools.hardware import HardwareAnalyzer\r\n\r\n\r\n# Load environment variables (DEEPSEEK_API_KEY, etc.)\r\nload_dotenv()\r\n\r\n\r\ndef setup_llm(api_key_override=None):\r\n \"\"\"Initialize ChatDeepSeek with structured output.\"\"\"\r\n # Priority: command line argument > environment variable\r\n api_key = api_key_override or os.getenv(\"DEEPSEEK_API_KEY\")\r\n if not api_key:\r\n print(\"\\n⚠️ Error: DEEPSEEK_API_KEY not set\")\r\n print(\"Please set using one of the following methods:\")\r\n print(\" 1. Command line: --api-key 'your-key'\")\r\n print(\" 2. Environment: $env:DEEPSEEK_API_KEY='your-key' (PowerShell)\")\r\n print(\" 3. Environment: export DEEPSEEK_API_KEY='your-key' (Bash)\")\r\n print(\"\\n或使用 --mock-llm 参数进行测试（无需 API）\")\r\n raise ValueError(\"DEEPSEEK_API_KEY not set in environment\")\r\n \r\n llm = ChatDeepSeek(\r\n model=\"deepseek-chat\",\r\n api_key=api_key,\r\n temperature=0.7,\r\n max_tokens=1000,\r\n )\r\n \r\n # Enable structured output with Pydantic schema\r\n # 输出严格对准 TuneDecision\r\n llm_with_schema = llm.with_structured_output(\r\n TuneDecision,\r\n method=\"json_schema\",\r\n strict=True\r\n )\r\n \r\n return llm_with_schema\r\n\r\n\r\n\r\n\r\ndef main():\r\n parser = argparse.ArgumentParser(description=\"ValTune-Agent CLI\")\r\n parser.add_argument(\r\n \"--base-config\",\r\n type=str,\r\n default=\"configs/base.yaml\",\r\n help=\"Path to base.yaml\"\r\n )\r\n parser.add_argument(\r\n \"--train-cmd\",\r\n type=str,\r\n required=True,\r\n help='Training command template, e.g., \"python train.py --cfg {config} --run_dir {run_dir}\"'\r\n )\r\n parser.add_argument(\r\n \"--max-rounds\",\r\n type=int,\r\n default=20,\r\n help=\"Maximum training rounds\"\r\n )\r\n parser.add_argument(\r\n \"--timeout\",\r\n type=int,\r\n default=None,\r\n help=\"Timeout per training round (seconds)\"\r\n )\r\n parser.add_argument(\r\n \"--simple-loop\",\r\n action=\"store_true\",\r\n help=\"Use simple loop instead of LangGraph workflow\"\r\n )\r\n parser.add_argument(\r\n \"--api-key\",\r\n type=str,\r\n default=\"sk-997fd694504445b293eb2eaede6c5853\",\r\n help=\"DeepSeek API key (overrides environment variable)\"\r\n )\r\n \r\n args = parser.parse_args()\r\n \r\n # Initialize components\r\n config_manager = ConfigManager(args.base_config)\r\n metrics_parser = MetricsParser()\r\n runner = TrainingRunner(args.train_cmd, config_manager)\r\n llm = setup_llm(api_key_override=args.api_key)\r\n \r\n # Load base config\r\n base_config = config_manager.load_yaml(args.base_config)\r\n \r\n print(f\"Starting ValTune-Agent loop (max {args.max_rounds} rounds)\")\r\n print(f\"Base config: {args.base_config}\")\r\n print(f\"Training command: {args.train_cmd}\")\r\n print(f\"Mode: {'Simple Loop' if args.simple_loop else 'LangGraph (Default)'}\")\r\n print(f\"LLM: DeepSeek API\\n\")\r\n \r\n # Default to LangGraph workflow unless --simple-loop is specified\r\n if not args.simple_loop:\r\n # Use LangGraph workflow (default mode)\r\n print(\"Using LangGraph workflow (default)...\\n\")\r\n \r\n # Initialize LangGraph with all components\r\n tuning_graph = TuningGraph(\r\n llm_client=llm,\r\n config_manager=config_manager,\r\n runner=runner,\r\n metrics_parser=metrics_parser\r\n )\r\n \r\n # Prepare initial state for workflow\r\n initial_state = {\r\n \"base_config_path\": args.base_config,\r\n \"current_config\": base_config,\r\n \"round_number\": 0,\r\n \"metrics_history\": [],\r\n \"decisions\": [],\r\n \"should_stop\": False,\r\n \"max_rounds\": args.max_rounds\r\n }\r\n \r\n # Execute the LangGraph workflow\r\n final_state = tuning_graph.run(initial_state)\r\n \r\n # Print final summary\r\n print(f\"\\n{'=' * 60}\")\r\n print(\"Tuning Loop Completed (LangGraph)\")\r\n print(f\"{'=' * 60}\")\r\n print(f\"Total rounds: {len(final_state.get('decisions', []))}\")\r\n metrics_history = final_state.get('metrics_history', [])\r\n if metrics_history:\r\n print(f\"Best mAP50: {max([m.get('map_50', 0) for m in metrics_history if m.get('map_50')], default=0):.4f}\")\r\n print(f\"Best mAP50-95: {max([m.get('map_50_95', 0) for m in metrics_history if m.get('map_50_95')], default=0):.4f}\")\r\n \r\n else:\r\n # Use simple sequential loop mode\r\n print(\"Using simple loop...\\n\")\r\n \r\n # Track state across rounds: metrics and decisions\r\n state = {\r\n \"round_number\": 0,\r\n \"base_config\": base_config,\r\n \"metrics_history\": [],\r\n \"decisions\": [],\r\n }\r\n \r\n for round_num in range(args.max_rounds):\r\n print(f\"\\n{'=' * 60}\")\r\n print(f\"Round {round_num}\")\r\n print(f\"{'=' * 60}\")\r\n \r\n state[\"round_number\"] = round_num\r\n \r\n # 1. Launch training with current configuration\r\n config = state[\"base_config\"]\r\n print(f\"Launching training with config:\")\r\n for k, v in config.items():\r\n print(f\" {k}: {v}\")\r\n \r\n run_dir = runner.launch_training(config, round_num, timeout=args.timeout)\r\n \r\n # 2. Parse metrics from training output\r\n metrics = metrics_parser.parse_metrics(str(run_dir))\r\n state[\"metrics_history\"].append(metrics)\r\n print(f\"\\nValidation Metrics:\")\r\n print(f\" mAP50: {metrics.map_50:.4f}\" if metrics.map_50 else \" mAP50: N/A\")\r\n print(f\" mAP50-95: {metrics.map_50_95:.4f}\" if metrics.map_50_95 else \" mAP50-95: N/A\")\r\n print(f\" Precision: {metrics.precision:.4f}\" if metrics.precision else \" Precision: N/A\")\r\n print(f\" Recall: {metrics.recall:.4f}\" if metrics.recall else \" Recall: N/A\")\r\n print(f\" Val Loss: {metrics.val_loss:.4f}\" if metrics.val_loss else \" Val Loss: N/A\")\r\n \r\n # 3. Query LLM for optimization decision\r\n history_text = format_history(state[\"decisions\"], state[\"metrics_history\"])\r\n \r\n prompt = TUNING_PROMPT_TEMPLATE.format(\r\n round_number=round_num,\r\n history=history_text,\r\n map_50=metrics.map_50 or 0.0,\r\n map_50_95=metrics.map_50_95 or 0.0,\r\n precision=metrics.precision or 0.0,\r\n recall=metrics.recall or 0.0,\r\n val_loss=metrics.val_loss or 0.0,\r\n epoch=metrics.epoch,\r\n current_config=json.dumps(config, indent=2)\r\n )\r\n \r\n print(f\"\\nQuerying LLM for optimization suggestions...\")\r\n \r\n try:\r\n decision = llm.invoke(SYSTEM_PROMPT + \"\\n\\n\" + prompt)\r\n # Ensure decision is a TuneDecision object\r\n if not isinstance(decision, TuneDecision):\r\n decision = TuneDecision(**decision)\r\n state[\"decisions\"].append(decision)\r\n except Exception as e:\r\n print(f\"Error querying LLM: {e}\")\r\n break\r\n \r\n print(f\"\\nOptimization Decision:\")\r\n print(f\" Action: {decision.action}\")\r\n print(f\" Reason: {decision.reason}\")\r\n if decision.patch:\r\n print(f\" Suggested Changes: {decision.patch.dict(exclude_none=True)}\")\r\n \r\n # 4. Check if LLM recommends stopping\r\n if decision.action == \"stop\":\r\n print(f\"\\nConvergence detected - stopping tuning.\")\r\n break\r\n \r\n # 5. Apply suggested changes for next round\r\n if decision.patch:\r\n state[\"base_config\"] = config_manager.apply_patch(\r\n state[\"base_config\"],\r\n decision.patch\r\n )\r\n print(f\"\\nConfiguration updated for next round\")\r\n \r\n # Record this round's metadata\r\n runner.save_round_metadata(\r\n round_num,\r\n config,\r\n metrics.dict(),\r\n decision.dict() if hasattr(decision, 'dict') else decision\r\n )\r\n \r\n # Print final optimization summary\r\n print(f\"\\n{'=' * 60}\")\r\n print(\"Hyperparameter Tuning Complete\")\r\n print(f\"{'=' * 60}\")\r\n print(f\"Total optimization rounds: {len(state['decisions'])}\")\r\n print(f\"Best mAP50: {max([m.map_50 for m in state['metrics_history'] if m.map_50], default=0):.4f}\")\r\n print(f\"Best mAP50-95: {max([m.map_50_95 for m in state['metrics_history'] if m.map_50_95], default=0):.4f}\")\r\n\r\n\r\nif __name__ == \"__main__\":\r\n main()\r\n"}}}} |