{"repo": "bytedance/deer-flow", "files": {"backend/tests/test_setup_wizard.py": {"459dd78707c7082662b49a4728e95d52d631d26a": {"t": 1785595300.1251097, "n": 31034, "h": "35d07eb8f537a0b8", "c": "\"\"\"Unit tests for the Setup Wizard (scripts/wizard/).\n\nRun from repo root:\n cd backend && uv run pytest tests/test_setup_wizard.py -v\n\"\"\"\n\nfrom __future__ import annotations\n\nimport yaml\nfrom wizard import ui as wizard_ui\nfrom wizard.providers import LLM_PROVIDERS, SEARCH_PROVIDERS, WEB_FETCH_PROVIDERS, LLMProvider, with_thinking_support\nfrom wizard.steps import channels as channels_step\nfrom wizard.steps import llm as llm_step\nfrom wizard.steps import search as search_step\nfrom wizard.writer import (\n build_minimal_config,\n read_env_file,\n write_config_yaml,\n write_env_file,\n)\n\n\nclass TestProviders:\n def test_llm_providers_not_empty(self):\n assert len(LLM_PROVIDERS) >= 8\n\n def test_llm_providers_cover_config_example_families(self):\n providers = {provider.name: provider for provider in LLM_PROVIDERS}\n\n expected = {\n \"volcengine\",\n \"openai\",\n \"openai_responses\",\n \"ollama_qwen\",\n \"ollama_gemma\",\n \"anthropic\",\n \"google\",\n \"gemini_openai_gateway\",\n \"mimo\",\n \"deepseek\",\n \"kimi\",\n \"novita\",\n \"minimax\",\n \"minimax_cn\",\n \"openrouter\",\n \"vllm\",\n \"mindie\",\n \"codex\",\n \"claude_code\",\n }\n assert expected.issubset(providers)\n\n assert providers[\"openai_responses\"].extra_config[\"use_responses_api\"] is True\n assert providers[\"gemini_openai_gateway\"].use == \"deerflow.models.patched_openai:PatchedChatOpenAI\"\n assert providers[\"mimo\"].use == \"deerflow.models.patched_mimo:PatchedChatMiMo\"\n assert providers[\"deepseek\"].use == \"deerflow.models.patched_deepseek:PatchedChatDeepSeek\"\n assert providers[\"volcengine\"].extra_config[\"api_base\"] == \"https://ark.cn-beijing.volces.com/api/v3\"\n\n def test_minimax_vision_is_per_model(self):\n \"\"\"M3 supports vision; M2.7 variants are text-only.\n\n The provider-level extra_config carries the default (M3) capability, but\n extra_config_for() must drop vision when an M2.7 model is selected.\n \"\"\"\n providers = {provider.name: provider for provider in LLM_PROVIDERS}\n\n for name in (\"minimax\", \"minimax_cn\"):\n provider = providers[name]\n assert provider.extra_config[\"supports_vision\"] is True\n assert provider.extra_config_for(\"MiniMax-M3\")[\"supports_vision\"] is True\n assert provider.extra_config_for(\"MiniMax-M2.7\")[\"supports_vision\"] is False\n assert provider.extra_config_for(\"MiniMax-M2.7-highspeed\")[\"supports_vision\"] is False\n # Override must not mutate the shared provider-level config.\n assert provider.extra_config[\"supports_vision\"] is True\n\n def test_extra_config_for_returns_provider_config_without_override(self):\n \"\"\"Providers without per-model overrides return their config unchanged.\"\"\"\n providers = {provider.name: provider for provider in LLM_PROVIDERS}\n openai = providers[\"openai\"]\n assert openai.extra_config_for(\"gpt-5\") == openai.extra_config\n\n def test_llm_providers_have_required_fields(self):\n for p in LLM_PROVIDERS:\n assert p.name\n assert p.display_name\n assert p.use\n assert \":\" in p.use, f\"Provider '{p.name}' use path must contain ':'\"\n assert p.models\n assert p.default_model in p.models\n\n def test_search_providers_have_required_fields(self):\n for sp in SEARCH_PROVIDERS:\n assert sp.name\n assert sp.display_name\n assert sp.use\n assert \":\" in sp.use\n\n def test_search_and_fetch_include_firecrawl(self):\n assert any(provider.name == \"firecrawl\" for provider in SEARCH_PROVIDERS)\n assert any(provider.name == \"firecrawl\" for provider in WEB_FETCH_PROVIDERS)\n\n def test_web_fetch_providers_have_required_fields(self):\n for provider in WEB_FETCH_PROVIDERS:\n assert provider.name\n assert provider.display_name\n assert provider.use\n assert \":\" in provider.use\n assert provider.tool_name == \"web_fetch\"\n\n def test_at_least_one_free_search_provider(self):\n \"\"\"At least one search provider needs no API key.\"\"\"\n free = [sp for sp in SEARCH_PROVIDERS if sp.env_var is None]\n assert free, \"Expected at least one free (no-key) search provider\"\n\n def test_at_least_one_free_web_fetch_provider(self):\n free = [provider for provider in WEB_FETCH_PROVIDERS if provider.env_var is None]\n assert free, \"Expected at least one free (no-key) web fetch provider\"\n\n\nclass TestBuildMinimalConfig:\n def test_produces_valid_yaml(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI / gpt-4o\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n )\n data = yaml.safe_load(content)\n assert data is not None\n assert \"models\" in data\n assert len(data[\"models\"]) == 1\n model = data[\"models\"][0]\n assert model[\"name\"] == \"gpt-4o\"\n assert model[\"use\"] == \"langchain_openai:ChatOpenAI\"\n assert model[\"model\"] == \"gpt-4o\"\n assert model[\"api_key\"] == \"$OPENAI_API_KEY\"\n\n def test_gemini_uses_gemini_api_key_field(self):\n content = build_minimal_config(\n provider_use=\"langchain_google_genai:ChatGoogleGenerativeAI\",\n model_name=\"gemini-2.0-flash\",\n display_name=\"Gemini\",\n api_key_field=\"gemini_api_key\",\n env_var=\"GEMINI_API_KEY\",\n )\n data = yaml.safe_load(content)\n model = data[\"models\"][0]\n assert \"gemini_api_key\" in model\n assert model[\"gemini_api_key\"] == \"$GEMINI_API_KEY\"\n assert \"api_key\" not in model\n\n def test_search_tool_included(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n search_use=\"deerflow.community.tavily.tools:web_search_tool\",\n search_extra_config={\"max_results\": 5},\n )\n data = yaml.safe_load(content)\n search_tool = next(t for t in data.get(\"tools\", []) if t[\"name\"] == \"web_search\")\n assert search_tool[\"max_results\"] == 5\n\n def test_openrouter_defaults_are_preserved(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"google/gemini-2.5-flash-preview\",\n display_name=\"OpenRouter\",\n api_key_field=\"api_key\",\n env_var=\"OPENROUTER_API_KEY\",\n extra_model_config={\n \"base_url\": \"https://openrouter.ai/api/v1\",\n \"request_timeout\": 600.0,\n \"max_retries\": 2,\n \"max_tokens\": 8192,\n \"temperature\": 0.7,\n },\n )\n data = yaml.safe_load(content)\n model = data[\"models\"][0]\n assert model[\"base_url\"] == \"https://openrouter.ai/api/v1\"\n assert model[\"request_timeout\"] == 600.0\n assert model[\"max_retries\"] == 2\n assert model[\"max_tokens\"] == 8192\n assert model[\"temperature\"] == 0.7\n\n def test_web_fetch_tool_included(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n web_fetch_use=\"deerflow.community.jina_ai.tools:web_fetch_tool\",\n web_fetch_extra_config={\"timeout\": 10},\n )\n data = yaml.safe_load(content)\n fetch_tool = next(t for t in data.get(\"tools\", []) if t[\"name\"] == \"web_fetch\")\n assert fetch_tool[\"timeout\"] == 10\n\n def test_no_search_tool_when_not_configured(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n )\n data = yaml.safe_load(content)\n tool_names = [t[\"name\"] for t in data.get(\"tools\", [])]\n assert \"web_search\" not in tool_names\n assert \"web_fetch\" not in tool_names\n\n def test_sandbox_included(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n )\n data = yaml.safe_load(content)\n assert \"sandbox\" in data\n assert \"use\" in data[\"sandbox\"]\n assert data[\"sandbox\"][\"use\"] == \"deerflow.sandbox.local:LocalSandboxProvider\"\n assert data[\"sandbox\"][\"allow_host_bash\"] is False\n\n def test_bash_tool_disabled_by_default(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n )\n data = yaml.safe_load(content)\n tool_names = [t[\"name\"] for t in data.get(\"tools\", [])]\n assert \"bash\" not in tool_names\n\n def test_can_enable_container_sandbox_and_bash(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n sandbox_use=\"deerflow.community.aio_sandbox:AioSandboxProvider\",\n include_bash_tool=True,\n )\n data = yaml.safe_load(content)\n assert data[\"sandbox\"][\"use\"] == \"deerflow.community.aio_sandbox:AioSandboxProvider\"\n assert \"allow_host_bash\" not in data[\"sandbox\"]\n tool_names = [t[\"name\"] for t in data.get(\"tools\", [])]\n assert \"bash\" in tool_names\n\n def test_can_disable_write_tools(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n include_write_tools=False,\n )\n data = yaml.safe_load(content)\n tool_names = [t[\"name\"] for t in data.get(\"tools\", [])]\n assert \"write_file\" not in tool_names\n assert \"str_replace\" not in tool_names\n\n def test_config_version_present(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n config_version=5,\n )\n data = yaml.safe_load(content)\n assert data[\"config_version\"] == 5\n\n def test_cli_provider_does_not_emit_fake_api_key(self):\n content = build_minimal_config(\n provider_use=\"deerflow.models.openai_codex_provider:CodexChatModel\",\n model_name=\"gpt-5.4\",\n display_name=\"Codex CLI\",\n api_key_field=\"api_key\",\n env_var=None,\n )\n data = yaml.safe_load(content)\n model = data[\"models\"][0]\n assert \"api_key\" not in model\n\n def test_responses_api_provider_defaults_are_preserved(self):\n provider = next(p for p in LLM_PROVIDERS if p.name == \"openai_responses\")\n content = build_minimal_config(\n provider_use=provider.use,\n model_name=provider.default_model,\n display_name=provider.display_name,\n api_key_field=provider.api_key_field,\n env_var=provider.env_var,\n extra_model_config=provider.extra_config,\n )\n data = yaml.safe_load(content)\n model = data[\"models\"][0]\n assert model[\"use_responses_api\"] is True\n assert model[\"output_version\"] == \"responses/v1\"\n assert model[\"supports_vision\"] is True\n\n def test_patched_thinking_provider_defaults_are_preserved(self):\n provider = next(p for p in LLM_PROVIDERS if p.name == \"mimo\")\n content = build_minimal_config(\n provider_use=provider.use,\n model_name=provider.default_model,\n display_name=provider.display_name,\n api_key_field=provider.api_key_field,\n env_var=provider.env_var,\n extra_model_config=provider.extra_config,\n )\n data = yaml.safe_load(content)\n model = data[\"models\"][0]\n assert model[\"use\"] == \"deerflow.models.patched_mimo:PatchedChatMiMo\"\n assert model[\"base_url\"] == \"https://api.xiaomimimo.com/v1\"\n assert model[\"api_key\"] == \"$MIMO_API_KEY\"\n assert model[\"supports_thinking\"] is True\n assert model[\"when_thinking_enabled\"][\"extra_body\"][\"thinking\"][\"type\"] == \"enabled\"\n assert model[\"when_thinking_disabled\"][\"extra_body\"][\"thinking\"][\"type\"] == \"disabled\"\n\n def test_can_enable_selected_channel_connections(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n channel_connection_providers=[\"feishu\", \"slack\"],\n )\n\n data = yaml.safe_load(content)\n channel_connections = data[\"channel_connections\"]\n\n assert channel_connections[\"enabled\"] is True\n assert channel_connections[\"feishu\"][\"enabled\"] is True\n assert channel_connections[\"slack\"][\"enabled\"] is True\n assert channel_connections[\"telegram\"][\"enabled\"] is False\n assert channel_connections[\"discord\"][\"enabled\"] is False\n assert channel_connections[\"dingtalk\"][\"enabled\"] is False\n assert channel_connections[\"wechat\"][\"enabled\"] is False\n assert channel_connections[\"wecom\"][\"enabled\"] is False\n\n def test_channel_connections_disabled_when_no_channels_selected(self):\n content = build_minimal_config(\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n channel_connection_providers=[],\n )\n\n data = yaml.safe_load(content)\n channel_connections = data[\"channel_connections\"]\n\n assert channel_connections[\"enabled\"] is False\n assert all(not config[\"enabled\"] for provider, config in channel_connections.items() if provider != \"enabled\")\n\n\nclass TestThinkingSupport:\n def test_other_provider_requests_thinking_prompt(self):\n other = next(p for p in LLM_PROVIDERS if p.name == \"other\")\n assert other.ask_thinking_support is True\n\n def test_with_thinking_support_enabled_wires_toggles(self):\n other = next(p for p in LLM_PROVIDERS if p.name == \"other\")\n original = dict(other.extra_config)\n\n updated = with_thinking_support(other, True)\n\n assert updated.extra_config[\"supports_thinking\"] is True\n assert updated.extra_config[\"when_thinking_enabled\"][\"extra_body\"][\"thinking\"][\"type\"] == \"enabled\"\n assert updated.extra_config[\"when_thinking_disabled\"][\"extra_body\"][\"thinking\"][\"type\"] == \"disabled\"\n # The shared provider singleton must not be mutated.\n assert other.extra_config == original\n\n def test_with_thinking_support_disabled_marks_unsupported(self):\n other = next(p for p in LLM_PROVIDERS if p.name == \"other\")\n\n updated = with_thinking_support(other, False)\n\n assert updated.extra_config[\"supports_thinking\"] is False\n assert \"when_thinking_enabled\" not in updated.extra_config\n\n\nclass TestLLMStep:\n def test_model_selection_defaults_to_provider_default_model(self, monkeypatch):\n provider = LLMProvider(\n name=\"test\",\n display_name=\"Test\",\n description=\"provider\",\n use=\"langchain_openai:ChatOpenAI\",\n models=[\"first-model\", \"default-model\"],\n default_model=\"default-model\",\n env_var=\"TEST_API_KEY\",\n package=\"langchain-openai\",\n )\n prompts: list[tuple[str, int | None]] = []\n\n def fake_choice(prompt, options, default=None):\n prompts.append((prompt, default))\n return default if default is not None else 0\n\n monkeypatch.setattr(llm_step, \"LLM_PROVIDERS\", [provider])\n monkeypatch.setattr(llm_step, \"ask_choice\", fake_choice)\n monkeypatch.setattr(llm_step, \"ask_secret\", lambda _prompt: \"key\")\n monkeypatch.setattr(llm_step, \"print_header\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_info\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_success\", lambda *_args, **_kwargs: None)\n\n result = llm_step.run_llm_step()\n\n assert result.model_name == \"default-model\"\n assert prompts == [(\"Enter choice\", None), (\"Select model\", 1)]\n\n def test_base_url_prompt_is_used_for_custom_gateway(self, monkeypatch):\n provider = LLMProvider(\n name=\"gateway\",\n display_name=\"Gateway\",\n description=\"provider\",\n use=\"langchain_openai:ChatOpenAI\",\n models=[\"gateway/model\"],\n default_model=\"gateway/model\",\n env_var=\"GATEWAY_API_KEY\",\n package=\"langchain-openai\",\n base_url_prompt=\"Gateway URL\",\n )\n\n monkeypatch.setattr(llm_step, \"LLM_PROVIDERS\", [provider])\n monkeypatch.setattr(llm_step, \"ask_choice\", lambda *_args, **_kwargs: 0)\n monkeypatch.setattr(llm_step, \"ask_text\", lambda *_args, **_kwargs: \"https://gateway.example/v1\")\n monkeypatch.setattr(llm_step, \"ask_secret\", lambda _prompt: \"key\")\n monkeypatch.setattr(llm_step, \"print_header\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_info\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_success\", lambda *_args, **_kwargs: None)\n\n result = llm_step.run_llm_step()\n\n assert result.base_url == \"https://gateway.example/v1\"\n\n def test_other_gateway_prompts_and_enables_thinking(self, monkeypatch):\n provider = LLMProvider(\n name=\"other\",\n display_name=\"Other OpenAI-compatible\",\n description=\"Custom gateway\",\n use=\"langchain_openai:ChatOpenAI\",\n models=[\"gpt-4o\"],\n default_model=\"gpt-4o\",\n env_var=\"OPENAI_API_KEY\",\n package=\"langchain-openai\",\n base_url_prompt=\"Base URL\",\n model_prompt=\"Model name\",\n ask_thinking_support=True,\n )\n\n monkeypatch.setattr(llm_step, \"LLM_PROVIDERS\", [provider])\n monkeypatch.setattr(llm_step, \"ask_choice\", lambda *_args, **_kwargs: 0)\n monkeypatch.setattr(llm_step, \"ask_text\", lambda *_args, **_kwargs: \"custom-thinking-model\")\n monkeypatch.setattr(llm_step, \"ask_secret\", lambda _prompt: \"key\")\n monkeypatch.setattr(llm_step, \"ask_yes_no\", lambda *_args, **_kwargs: True)\n monkeypatch.setattr(llm_step, \"print_header\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_info\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_success\", lambda *_args, **_kwargs: None)\n\n result = llm_step.run_llm_step()\n\n assert result.model_name == \"custom-thinking-model\"\n assert result.provider.extra_config[\"supports_thinking\"] is True\n assert result.provider.extra_config[\"when_thinking_enabled\"][\"extra_body\"][\"thinking\"][\"type\"] == \"enabled\"\n\n def test_other_gateway_declined_thinking_marks_unsupported(self, monkeypatch):\n provider = LLMProvider(\n name=\"other\",\n display_name=\"Other OpenAI-compatible\",\n description=\"Custom gateway\",\n use=\"langchain_openai:ChatOpenAI\",\n models=[\"gpt-4o\"],\n default_model=\"gpt-4o\",\n env_var=\"OPENAI_API_KEY\",\n package=\"langchain-openai\",\n base_url_prompt=\"Base URL\",\n model_prompt=\"Model name\",\n ask_thinking_support=True,\n )\n\n monkeypatch.setattr(llm_step, \"LLM_PROVIDERS\", [provider])\n monkeypatch.setattr(llm_step, \"ask_choice\", lambda *_args, **_kwargs: 0)\n monkeypatch.setattr(llm_step, \"ask_text\", lambda *_args, **_kwargs: \"plain-model\")\n monkeypatch.setattr(llm_step, \"ask_secret\", lambda _prompt: \"key\")\n monkeypatch.setattr(llm_step, \"ask_yes_no\", lambda *_args, **_kwargs: False)\n monkeypatch.setattr(llm_step, \"print_header\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_info\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(llm_step, \"print_success\", lambda *_args, **_kwargs: None)\n\n result = llm_step.run_llm_step()\n\n assert result.provider.extra_config[\"supports_thinking\"] is False\n assert \"when_thinking_enabled\" not in result.provider.extra_config\n\n\nclass TestChannelsStep:\n def test_returns_selected_channel_keys(self, monkeypatch):\n monkeypatch.setattr(channels_step, \"print_header\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(channels_step, \"print_info\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(channels_step, \"print_success\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(channels_step, \"ask_multi_choice\", lambda *_args, **_kwargs: [0, 3, 6])\n\n result = channels_step.run_channels_step()\n\n assert result.enabled_providers == [\"telegram\", \"feishu\", \"wecom\"]\n\n def test_empty_selection_disables_channel_connections(self, monkeypatch):\n monkeypatch.setattr(channels_step, \"print_header\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(channels_step, \"print_info\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(channels_step, \"print_success\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(channels_step, \"ask_multi_choice\", lambda *_args, **_kwargs: [])\n\n result = channels_step.run_channels_step()\n\n assert result.enabled_providers == []\n\n\nclass TestWizardUi:\n def test_multi_choice_blank_requires_input_without_default(self, monkeypatch):\n answers = iter([\"\", \"2\"])\n monkeypatch.setattr(\"builtins.input\", lambda _prompt: next(answers))\n\n assert wizard_ui.ask_multi_choice(\"Pick\", [\"First\", \"Second\"], default=None) == [1]\n\n def test_multi_choice_blank_accepts_empty_default(self, monkeypatch):\n monkeypatch.setattr(\"builtins.input\", lambda _prompt: \"\")\n\n assert wizard_ui.ask_multi_choice(\"Pick\", [\"First\", \"Second\"], default=[]) == []\n\n\n# ---------------------------------------------------------------------------\n# writer.py — env file helpers\n# ---------------------------------------------------------------------------\n\n\nclass TestEnvFileHelpers:\n def test_write_and_read_new_file(self, tmp_path):\n env_file = tmp_path / \".env\"\n write_env_file(env_file, {\"OPENAI_API_KEY\": \"sk-test123\"})\n pairs = read_env_file(env_file)\n assert pairs[\"OPENAI_API_KEY\"] == \"sk-test123\"\n\n def test_update_existing_key(self, tmp_path):\n env_file = tmp_path / \".env\"\n env_file.write_text(\"OPENAI_API_KEY=old-key\\n\")\n write_env_file(env_file, {\"OPENAI_API_KEY\": \"new-key\"})\n pairs = read_env_file(env_file)\n assert pairs[\"OPENAI_API_KEY\"] == \"new-key\"\n # Should not duplicate\n content = env_file.read_text()\n assert content.count(\"OPENAI_API_KEY\") == 1\n\n def test_preserve_existing_keys(self, tmp_path):\n env_file = tmp_path / \".env\"\n env_file.write_text(\"TAVILY_API_KEY=tavily-val\\n\")\n write_env_file(env_file, {\"OPENAI_API_KEY\": \"sk-new\"})\n pairs = read_env_file(env_file)\n assert pairs[\"TAVILY_API_KEY\"] == \"tavily-val\"\n assert pairs[\"OPENAI_API_KEY\"] == \"sk-new\"\n\n def test_preserve_comments(self, tmp_path):\n env_file = tmp_path / \".env\"\n env_file.write_text(\"# My .env file\\nOPENAI_API_KEY=old\\n\")\n write_env_file(env_file, {\"OPENAI_API_KEY\": \"new\"})\n content = env_file.read_text()\n assert \"# My .env file\" in content\n\n def test_read_ignores_comments(self, tmp_path):\n env_file = tmp_path / \".env\"\n env_file.write_text(\"# comment\\nKEY=value\\n\")\n pairs = read_env_file(env_file)\n assert \"# comment\" not in pairs\n assert pairs[\"KEY\"] == \"value\"\n\n\n# ---------------------------------------------------------------------------\n# writer.py — write_config_yaml\n# ---------------------------------------------------------------------------\n\n\nclass TestWriteConfigYaml:\n def test_generated_config_loadable_by_appconfig(self, tmp_path):\n \"\"\"The generated config.yaml must be parseable (basic YAML validity).\"\"\"\n\n config_path = tmp_path / \"config.yaml\"\n write_config_yaml(\n config_path,\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI / gpt-4o\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n )\n assert config_path.exists()\n with open(config_path) as f:\n data = yaml.safe_load(f)\n assert isinstance(data, dict)\n assert \"models\" in data\n\n def test_copies_example_defaults_for_unconfigured_sections(self, tmp_path):\n example_path = tmp_path / \"config.example.yaml\"\n example_path.write_text(\n yaml.safe_dump(\n {\n \"config_version\": 5,\n \"log_level\": \"info\",\n \"token_usage\": {\"enabled\": True},\n \"tool_groups\": [{\"name\": \"web\"}, {\"name\": \"file:read\"}, {\"name\": \"file:write\"}, {\"name\": \"bash\"}],\n \"tools\": [\n {\n \"name\": \"web_search\",\n \"group\": \"web\",\n \"use\": \"deerflow.community.ddg_search.tools:web_search_tool\",\n \"max_results\": 5,\n },\n {\n \"name\": \"web_fetch\",\n \"group\": \"web\",\n \"use\": \"deerflow.community.jina_ai.tools:web_fetch_tool\",\n \"timeout\": 10,\n },\n {\n \"name\": \"image_search\",\n \"group\": \"web\",\n \"use\": \"deerflow.community.image_search.tools:image_search_tool\",\n \"max_results\": 5,\n },\n {\"name\": \"ls\", \"group\": \"file:read\", \"use\": \"deerflow.sandbox.tools:ls_tool\"},\n {\"name\": \"write_file\", \"group\": \"file:write\", \"use\": \"deerflow.sandbox.tools:write_file_tool\"},\n {\"name\": \"bash\", \"group\": \"bash\", \"use\": \"deerflow.sandbox.tools:bash_tool\"},\n ],\n \"sandbox\": {\n \"use\": \"deerflow.sandbox.local:LocalSandboxProvider\",\n \"allow_host_bash\": False,\n },\n \"summarization\": {\"max_tokens\": 2048},\n },\n sort_keys=False,\n )\n )\n\n config_path = tmp_path / \"config.yaml\"\n write_config_yaml(\n config_path,\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI / gpt-4o\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n )\n with open(config_path) as f:\n data = yaml.safe_load(f)\n\n assert data[\"log_level\"] == \"info\"\n assert data[\"token_usage\"][\"enabled\"] is True\n assert data[\"tool_groups\"][0][\"name\"] == \"web\"\n assert data[\"summarization\"][\"max_tokens\"] == 2048\n assert any(tool[\"name\"] == \"image_search\" and tool[\"max_results\"] == 5 for tool in data[\"tools\"])\n\n def test_config_version_read_from_example(self, tmp_path):\n \"\"\"write_config_yaml should read config_version from config.example.yaml if present.\"\"\"\n\n example_path = tmp_path / \"config.example.yaml\"\n example_path.write_text(\"config_version: 99\\n\")\n\n config_path = tmp_path / \"config.yaml\"\n write_config_yaml(\n config_path,\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"gpt-4o\",\n display_name=\"OpenAI\",\n api_key_field=\"api_key\",\n env_var=\"OPENAI_API_KEY\",\n )\n with open(config_path) as f:\n data = yaml.safe_load(f)\n assert data[\"config_version\"] == 99\n\n def test_model_base_url_from_extra_config(self, tmp_path):\n config_path = tmp_path / \"config.yaml\"\n write_config_yaml(\n config_path,\n provider_use=\"langchain_openai:ChatOpenAI\",\n model_name=\"google/gemini-2.5-flash-preview\",\n display_name=\"OpenRouter\",\n api_key_field=\"api_key\",\n env_var=\"OPENROUTER_API_KEY\",\n extra_model_config={\"base_url\": \"https://openrouter.ai/api/v1\"},\n )\n with open(config_path) as f:\n data = yaml.safe_load(f)\n assert data[\"models\"][0][\"base_url\"] == \"https://openrouter.ai/api/v1\"\n\n\nclass TestSearchStep:\n def test_reuses_api_key_for_same_provider(self, monkeypatch):\n monkeypatch.setattr(search_step, \"print_header\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(search_step, \"print_success\", lambda *_args, **_kwargs: None)\n monkeypatch.setattr(search_step, \"print_info\", lambda *_args, **_kwargs: None)\n\n choices = iter([3, 1])\n prompts: list[str] = []\n\n def fake_choice(_prompt, _options, default=0):\n return next(choices)\n\n def fake_secret(prompt):\n prompts.append(prompt)\n return \"shared-api-key\"\n\n monkeypatch.setattr(search_step, \"ask_choice\", fake_choice)\n monkeypatch.setattr(search_step, \"ask_secret\", fake_secret)\n\n result = search_step.run_search_step()\n\n assert result.search_provider is not None\n assert result.fetch_provider is not None\n assert result.search_provider.name == \"exa\"\n assert result.fetch_provider.name == \"exa\"\n assert result.search_api_key == \"shared-api-key\"\n assert result.fetch_api_key == \"shared-api-key\"\n assert prompts == [\"EXA_API_KEY\"]\n"}}, "backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py": {"7025ccee403d016cfaeeb6f011475a370ff81e0d": {"t": 1785674279.1788626, "n": 115342, "h": "72a9842ecddaaaf9", "c": "\"\"\"AIO Sandbox Provider — orchestrates sandbox lifecycle with pluggable backends.\n\nThis provider composes:\n- SandboxBackend: how sandboxes are provisioned (local container vs remote/K8s)\n\nThe provider itself handles:\n- In-process caching for fast repeated access\n- Idle timeout management\n- Graceful shutdown with signal handling\n- Mount computation (thread-specific, skills)\n\"\"\"\n\nimport asyncio\nimport atexit\nimport contextlib\nimport hashlib\nimport logging\nimport os\nimport signal\nimport threading\nimport time\nimport uuid\nfrom collections.abc import Callable\nfrom concurrent.futures import ThreadPoolExecutor\n\ntry:\n import fcntl\nexcept ImportError: # pragma: no cover - Windows fallback\n fcntl = None # type: ignore[assignment]\n import msvcrt\n\nfrom deerflow.community.warm_pool_lifecycle import (\n DEFAULT_IDLE_TIMEOUT,\n DEFAULT_REPLICAS,\n WarmPoolLifecycleMixin,\n)\nfrom deerflow.community.warm_pool_lifecycle import (\n IDLE_CHECK_INTERVAL as _SHARED_IDLE_CHECK_INTERVAL,\n)\nfrom deerflow.config import get_app_config\nfrom deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths, join_host_path\nfrom deerflow.integrations.lark_cli import INTEGRATION_ID as LARK_CLI_INTEGRATION_ID\nfrom deerflow.integrations.lark_cli import LARK_CLI_SANDBOX_CONFIG_DIR, LARK_CLI_SANDBOX_DATA_DIR, LARK_CLI_SANDBOX_RUNTIME_DIR, ensure_lark_cli_credential_tree, lark_skills_installed\nfrom deerflow.runtime.user_context import get_effective_user_id\nfrom deerflow.sandbox.sandbox import Sandbox\nfrom deerflow.sandbox.sandbox_provider import SandboxProvider\n\nfrom .aio_sandbox import AioSandbox\nfrom .backend import SandboxBackend, wait_for_sandbox_ready, wait_for_sandbox_ready_async\nfrom .local_backend import LocalContainerBackend\nfrom .ownership import (\n OwnershipBackendError,\n RenewOutcome,\n SandboxOwnershipStore,\n compute_lease_ttl,\n generate_owner_id,\n make_sandbox_ownership_store,\n resolve_ownership_config,\n)\nfrom .remote_backend import RemoteSandboxBackend\nfrom .sandbox_info import SandboxInfo\n\nlogger = logging.getLogger(__name__)\n\n# Default configuration\nDEFAULT_IMAGE = \"enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest\"\nDEFAULT_PORT = 8080\nDEFAULT_CONTAINER_PREFIX = \"deer-flow-sandbox\"\nIDLE_CHECK_INTERVAL = _SHARED_IDLE_CHECK_INTERVAL\nTHREAD_LOCK_EXECUTOR_WORKERS = min(32, (os.cpu_count() or 1) + 4)\n_THREAD_LOCK_EXECUTOR = ThreadPoolExecutor(max_workers=THREAD_LOCK_EXECUTOR_WORKERS, thread_name_prefix=\"sandbox-lock-wait\")\natexit.register(_THREAD_LOCK_EXECUTOR.shutdown, wait=False, cancel_futures=True)\n\n\nclass SandboxBeingDestroyedError(RuntimeError):\n \"\"\"A peer is tearing this container down, so it must not be handed out.\n\n Raised on the acquire path when the ownership lease is in its teardown state.\n The caller drops the container from tracking and lets the normal\n discover-or-create path provision a fresh one, rather than handing an agent a\n sandbox that is about to stop underneath it.\n \"\"\"\n\n def __init__(self, sandbox_id: str) -> None:\n super().__init__(f\"sandbox {sandbox_id} is being destroyed by another instance\")\n self.sandbox_id = sandbox_id\n\n\nclass SandboxIdentityCollisionError(RuntimeError):\n \"\"\"A deterministic ID is already tracked for a different user/thread.\"\"\"\n\n def __init__(\n self,\n sandbox_id: str,\n stored_key: tuple[str, str] | None,\n requested_key: tuple[str, str],\n ) -> None:\n super().__init__(f\"sandbox ID collision for {sandbox_id}: tracked identity is {stored_key!r}, requested identity is {requested_key!r}\")\n self.sandbox_id = sandbox_id\n\n\ndef _lock_file_exclusive(lock_file) -> None:\n if fcntl is not None:\n fcntl.flock(lock_file, fcntl.LOCK_EX)\n return\n\n lock_file.seek(0)\n msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)\n\n\ndef _unlock_file(lock_file) -> None:\n if fcntl is not None:\n fcntl.flock(lock_file, fcntl.LOCK_UN)\n return\n\n lock_file.seek(0)\n msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)\n\n\ndef _open_lock_file(lock_path):\n return open(lock_path, \"a\", encoding=\"utf-8\")\n\n\nasync def _acquire_thread_lock_async(lock: threading.Lock) -> None:\n \"\"\"Acquire a threading.Lock without polling or using the default executor.\"\"\"\n loop = asyncio.get_running_loop()\n acquire_future = loop.run_in_executor(_THREAD_LOCK_EXECUTOR, lock.acquire, True)\n\n try:\n acquired = await asyncio.shield(acquire_future)\n except asyncio.CancelledError:\n acquire_future.add_done_callback(lambda task: _release_cancelled_lock_acquire(lock, task))\n raise\n\n if not acquired:\n raise RuntimeError(\"Failed to acquire sandbox thread lock\")\n\n\ndef _release_cancelled_lock_acquire(lock: threading.Lock, task: asyncio.Future[bool]) -> None:\n \"\"\"Release a lock acquired after its awaiting coroutine was cancelled.\"\"\"\n if task.cancelled():\n return\n\n try:\n acquired = task.result()\n except Exception as e:\n logger.warning(f\"Cancelled sandbox lock acquisition finished with error: {e}\")\n return\n\n if acquired:\n lock.release()\n\n\nclass AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):\n \"\"\"Sandbox provider that manages containers running the AIO sandbox.\n\n Architecture:\n This provider composes a SandboxBackend (how to provision), enabling:\n - Local Docker/Apple Container mode (auto-start containers)\n - Remote/K8s mode (connect to pre-existing sandbox URL)\n\n Configuration options in config.yaml under sandbox:\n use: deerflow.community.aio_sandbox:AioSandboxProvider\n image: \n port: 8080 # Base port for local containers\n container_prefix: deer-flow-sandbox\n idle_timeout: 600 # Idle timeout in seconds (0 to disable)\n replicas: 3 # Max concurrent sandbox containers (LRU eviction when exceeded)\n thread_data_mounts: null # null = backend auto-detection\n mounts: # Volume mounts for local containers\n - host_path: /path/on/host\n container_path: /path/in/container\n read_only: false\n environment: # Environment variables for containers\n NODE_ENV: production\n API_KEY: $MY_API_KEY\n \"\"\"\n\n # How long `_held_teardown_lease` waits for its heartbeat thread to exit\n # before deferring the final lease release to that (still-running) thread.\n # The store's socket timeout bounds each operation, but context exit can\n # catch the heartbeat in one final refresh and must then wait for its final\n # release. Keep this above both sequential five-second operation bounds so a\n # normally timing-out refresh + release still finishes synchronously.\n _TEARDOWN_JOIN_TIMEOUT_SECONDS = 12.0\n\n def __init__(self):\n self._lock = threading.Lock()\n self._sandboxes: dict[str, AioSandbox] = {} # sandbox_id -> AioSandbox instance\n self._sandbox_infos: dict[str, SandboxInfo] = {} # sandbox_id -> SandboxInfo (for destroy)\n self._thread_sandboxes: dict[tuple[str, str], str] = {} # (user_id, thread_id) -> sandbox_id\n self._thread_locks: dict[tuple[str, str], threading.Lock] = {} # (user_id, thread_id) -> in-process lock\n self._last_activity: dict[str, float] = {} # sandbox_id -> last activity timestamp\n # Warm pool: released sandboxes whose containers are still running.\n # Maps sandbox_id -> (SandboxInfo, release_timestamp).\n # Containers here can be reclaimed quickly (no cold-start) or destroyed\n # when replicas capacity is exhausted.\n self._warm_pool: dict[str, tuple[SandboxInfo, float]] = {}\n self._active_sandbox_identity: dict[str, tuple[str, str] | None] = {}\n self._warm_pool_identity: dict[str, tuple[str, str] | None] = {}\n # sandbox_id -> when reconciliation first saw it running with no lease.\n # Gates adoption behind a recovery grace (see _adoptable_after_grace).\n self._unowned_since: dict[str, float] = {}\n # The two halves of same-process exclusion. The ownership store excludes\n # peers and nothing else — `claim()` and `take()` both succeed against\n # our own lease by design — so `del:` says nothing to this process's own\n # threads. See _reserve_local_teardown / _acquire_epoch.\n self._local_teardown: set[str] = set()\n self._acquire_epoch: dict[str, int] = {}\n self._acquire_epoch_counter = 0\n self._acquire_inflight: dict[str, int] = {}\n self._shutdown_called = False\n self._idle_checker_stop = threading.Event()\n self._idle_checker_thread: threading.Thread | None = None\n self._renewal_stop = threading.Event()\n self._renewal_thread: threading.Thread | None = None\n # Per-instance id used for cross-instance sandbox ownership leases (#4206).\n self._owner_id = generate_owner_id()\n\n self._config = self._load_config()\n self._ownership_config = resolve_ownership_config(self._config.get(\"ownership\"), stream_bridge=self._config.get(\"stream_bridge\"))\n self._ownership: SandboxOwnershipStore = make_sandbox_ownership_store(self._ownership_config, owner_id=self._owner_id)\n if not self._ownership.supports_cross_process:\n # Peers cannot see these leases, so every container looks like an\n # orphan to them. Say so once rather than letting #4206 resurface\n # silently on a multi-worker deployment that never set the config.\n logger.warning(\n \"Sandbox ownership store cannot coordinate across processes (sandbox.ownership.type: %s). \"\n \"Safe for a single gateway instance only — multi-worker / load-balanced gateways sharing a \"\n \"container backend must set sandbox.ownership.type: redis, or peers will adopt and idle-destroy \"\n \"each other's live sandboxes (#4206).\",\n self._ownership_config.type,\n )\n self._backend: SandboxBackend = self._create_backend()\n\n # Register shutdown handler\n atexit.register(self.shutdown)\n self._register_signal_handlers()\n\n # Reconcile orphaned containers from previous process lifecycles\n self._reconcile_orphans()\n\n # Renewal is independent of idle cleanup: an owner must keep proving it is\n # alive even when the idle reaper is disabled, or peers adopt its live\n # containers once the lease lapses (idle_timeout: 0 is a supported config).\n self._start_lease_renewal()\n\n # Start idle checker if enabled\n if self._config.get(\"idle_timeout\", DEFAULT_IDLE_TIMEOUT) > 0:\n self._start_idle_checker()\n\n @property\n def uses_thread_data_mounts(self) -> bool:\n \"\"\"Whether thread workspace/uploads/outputs are visible via mounts.\n\n Local container backends bind-mount the thread data directories, so files\n written by the gateway are already visible when the sandbox starts.\n Remote backends may require explicit file sync. Operators can override\n this detection when gateway and remote sandboxes share the same storage.\n \"\"\"\n override = self._config.get(\"thread_data_mounts\")\n if override is not None:\n return override\n return isinstance(self._backend, LocalContainerBackend)\n\n # ── Factory methods ──────────────────────────────────────────────────\n\n def _create_backend(self) -> SandboxBackend:\n \"\"\"Create the appropriate backend based on configuration.\n\n Selection logic (checked in order):\n 1. ``provisioner_url`` set → RemoteSandboxBackend (provisioner mode)\n Provisioner dynamically creates Pods + Services in k3s.\n 2. Default → LocalContainerBackend (local mode)\n Local provider manages container lifecycle directly (start/stop).\n \"\"\"\n provisioner_url = self._config.get(\"provisioner_url\")\n if provisioner_url:\n logger.info(f\"Using remote sandbox backend with provisioner at {provisioner_url}\")\n api_key = self._config.get(\"provisioner_api_key\", \"\")\n return RemoteSandboxBackend(provisioner_url=provisioner_url, api_key=api_key)\n\n logger.info(\"Using local container sandbox backend\")\n return LocalContainerBackend(\n image=self._config[\"image\"],\n base_port=self._config[\"port\"],\n container_prefix=self._config[\"container_prefix\"],\n config_mounts=self._config[\"mounts\"],\n environment=self._config[\"environment\"],\n )\n\n # ── Configuration ────────────────────────────────────────────────────\n\n def _load_config(self) -> dict:\n \"\"\"Load sandbox configuration from app config.\"\"\"\n config = get_app_config()\n sandbox_config = config.sandbox\n\n idle_timeout = getattr(sandbox_config, \"idle_timeout\", None)\n replicas = getattr(sandbox_config, \"replicas\", None)\n\n return {\n \"image\": sandbox_config.image or DEFAULT_IMAGE,\n \"port\": sandbox_config.port or DEFAULT_PORT,\n \"container_prefix\": sandbox_config.container_prefix or DEFAULT_CONTAINER_PREFIX,\n \"idle_timeout\": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,\n \"replicas\": replicas if replicas is not None else DEFAULT_REPLICAS,\n \"mounts\": sandbox_config.mounts or [],\n \"thread_data_mounts\": getattr(sandbox_config, \"thread_data_mounts\", None),\n \"environment\": self._resolve_env_vars(sandbox_config.environment or {}),\n \"ownership\": getattr(sandbox_config, \"ownership\", None),\n # A redis stream bridge means the deployment is multi-instance, which\n # is what the ownership store must default to. Read the same source\n # the bridge's own resolver reads, not just its env var.\n \"stream_bridge\": getattr(config, \"stream_bridge\", None),\n # provisioner URL for dynamic pod management (e.g. http://provisioner:8002)\n \"provisioner_url\": getattr(sandbox_config, \"provisioner_url\", None) or \"\",\n \"provisioner_api_key\": getattr(sandbox_config, \"provisioner_api_key\", None) or \"\",\n }\n\n @staticmethod\n def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:\n \"\"\"Resolve environment variable references (values starting with $).\"\"\"\n resolved = {}\n for key, value in env_config.items():\n if isinstance(value, str) and value.startswith(\"$\"):\n env_name = value[1:]\n resolved[key] = os.environ.get(env_name, \"\")\n else:\n resolved[key] = str(value)\n return resolved\n\n # ── Cross-instance ownership leases ───────────────────────────────────\n\n def _publish_ownership(self, sandbox_id: str) -> None:\n \"\"\"Take responsibility for *sandbox_id* on the acquire path.\n\n Takes over from whichever instance served this thread last — the\n container is deterministic per (user, thread), so a turn routing here is\n a legitimate handover. The previous owner's next renewal reports LOST and\n it stops tracking the container without touching it.\n\n Deliberately **not** fail-open. Swallowing the error and handing the\n sandbox out anyway would leave it unowned while in active use, so a peer\n would see an orphan and reap it — the exact failure this store exists to\n stop. Callers must let this propagate.\n\n The intent mark is published **before** the round trip, and that ordering\n is the point. ``take()`` makes the takeover durable before it returns —\n on redis the server has committed the SET while the reply is still in\n flight — so bumping the epoch afterwards leaves a window where the store\n already says the container is ours but the guard still reads as though it\n were not. A renewal holding an older ``LOST`` walks straight through it,\n drops the maps, and closes the client this call is about to hand back, so\n acquire returns an id the provider no longer tracks and ``get()`` answers\n ``None``. A guard must become visible no later than the transition it\n guards; the epoch alone cannot, because it can only be written after the\n call that performs the transition returns.\n\n So the two marks cover the two halves and are both needed: the intent\n mark covers \"an acquire is mid-flight\", the epoch covers \"an acquire\n completed since you decided\".\n\n Raises:\n SandboxBeingDestroyedError: a peer is tearing this container down, so\n it must not be handed to an agent (the destroy → re-acquire race).\n OwnershipBackendError: ownership could not be published.\n \"\"\"\n with self._lock:\n self._acquire_inflight[sandbox_id] = self._acquire_inflight.get(sandbox_id, 0) + 1\n try:\n if not self._ownership.take(sandbox_id):\n raise SandboxBeingDestroyedError(sandbox_id)\n with self._lock:\n self._acquire_epoch_counter += 1\n self._acquire_epoch[sandbox_id] = self._acquire_epoch_counter\n finally:\n # A count rather than a set: acquires for one id are serialized by\n # the per-thread lock today, so a set would be equivalent — but that\n # is an assumption about a caller two layers up, and if it ever\n # stopped holding, a set would be cleared by the first finisher and\n # silently reopen this window. Counting removes the assumption.\n with self._lock:\n remaining = self._acquire_inflight.get(sandbox_id, 0) - 1\n if remaining > 0:\n self._acquire_inflight[sandbox_id] = remaining\n else:\n self._acquire_inflight.pop(sandbox_id, None)\n\n # ── Same-process exclusion (the half the store does not provide) ───────\n #\n # A lease excludes *peers*: `claim()` succeeds against our own `own:` lease\n # by design (that is what lets a destroy path claim what it already owns),\n # and `take()` succeeds against it too. So between this process's reaper\n # threads — idle checker, renewal, eviction — and its own acquire path, the\n # store provides **no exclusion at all**. Every reaper decides outside\n # `self._lock` (a store round trip must not be held under the lock that\n # guards every acquire), so each one acts on a decision an acquire may\n # already have invalidated. The two helpers below are that missing half, one\n # per direction:\n #\n # reaping — we are about to stop/drop it, so nothing may promote it:\n # reserve it, and make every promote path honour the reservation\n # exactly as it honours a peer's `del:`.\n # forgetting — a peer legitimately owns it and must win, so the promote is\n # the thing to detect: compare the acquire epoch we decided on.\n\n def _reserve_local_teardown(self, sandbox_id: str, still_reapable: Callable[[], bool]) -> bool:\n \"\"\"Reserve *sandbox_id* for teardown by this process.\n\n ``still_reapable`` is evaluated in the **same** critical section as the\n reservation, so no acquire can slip between the last check and the mark.\n That pairing is the whole point: checking first and marking second is the\n window, not a narrower version of it.\n\n Consequence, and the one rule a new caller has to know: **the predicate\n runs with ``self._lock`` held**, which is a plain ``Lock``, so a predicate\n that touches the lock — directly, or via a provider method that takes it —\n deadlocks. Predicates must be cheap, non-blocking reads of the maps\n (``sandbox_id in self._warm_pool``, a ``_last_activity`` comparison). The\n constraint is stated rather than engineered around on purpose: making the\n lock reentrant to tolerate it would trade a loud hang for a quiet class of\n re-entrancy bugs everywhere else in this provider.\n \"\"\"\n with self._lock:\n if sandbox_id in self._local_teardown or not still_reapable():\n return False\n self._local_teardown.add(sandbox_id)\n return True\n\n def _finish_local_teardown(self, sandbox_id: str) -> None:\n with self._lock:\n self._local_teardown.discard(sandbox_id)\n\n def _being_torn_down_locally(self, sandbox_id: str) -> bool:\n \"\"\"Whether a reaper thread in *this* process holds *sandbox_id*.\n\n Callers must already hold ``self._lock``.\n \"\"\"\n return sandbox_id in self._local_teardown\n\n def _acquire_epoch_of(self, sandbox_id: str) -> int:\n \"\"\"Snapshot the acquire generation, so a stale decision can be detected.\n\n Bumped only by ``_publish_ownership`` — i.e. exactly when an acquire path\n (re)takes the lease on the way to handing the sandbox to an agent.\n Re-establishing a lapsed lease from ``_refresh_ownership`` deliberately\n does not bump it: nothing was handed out, so a reaper's decision about\n that id is still current.\n \"\"\"\n with self._lock:\n return self._acquire_epoch.get(sandbox_id, 0)\n\n def _claim_ownership(self, sandbox_id: str, *, for_destroy: bool = False) -> bool:\n \"\"\"Take (or refresh) ownership of *sandbox_id*.\n\n A successful claim is what makes acting on the container safe: while we\n hold the lease a peer's claim fails. With ``for_destroy`` the lease is\n additionally marked as a teardown, which a concurrent acquire-side\n ``take()`` refuses — that is what closes the ownership-check → container-\n stop window the deleted per-sandbox flock guard used to cover.\n\n Fails closed on a backend error: ownership unknown is treated as\n \"not ours\" so we neither adopt nor destroy the container.\n \"\"\"\n try:\n return self._ownership.claim(sandbox_id, for_destroy=for_destroy)\n except OwnershipBackendError as e:\n logger.warning(\"Sandbox ownership claim failed for %s (treating as not owned): %s\", sandbox_id, e)\n return False\n\n def _release_ownership(self, sandbox_id: str) -> None:\n try:\n self._ownership.release(sandbox_id)\n except OwnershipBackendError as e:\n # Best effort: the lease expires on its own, so a failed release\n # delays reuse rather than corrupting ownership.\n logger.warning(\"Failed to release sandbox ownership for %s: %s\", sandbox_id, e)\n\n def _refresh_ownership(self, sandbox_id: str) -> bool:\n \"\"\"Keep holding *sandbox_id*'s lease. False when a peer has taken it.\n\n A **lapsed** lease is re-established rather than treated as lost: nobody\n holds it, so re-claiming is safe, and this is what keeps a Redis restart\n (which drops every key) from evicting every live sandbox fleet-wide. A\n lease a peer actually holds is never re-taken — that is the #4206 kill.\n \"\"\"\n try:\n outcome = self._ownership.renew(sandbox_id)\n except OwnershipBackendError as e:\n # Unknown, not lost: keep the sandbox and retry next tick. The TTL\n # still bounds how long a genuinely dead owner holds the lease.\n logger.warning(\"Could not renew sandbox ownership for %s, will retry: %s\", sandbox_id, e)\n return True\n\n if outcome is RenewOutcome.RENEWED:\n return True\n if outcome is RenewOutcome.LAPSED:\n # Free: re-establish. This is the deliberate fail-open renewal path,\n # so it cannot use `_claim_ownership`: that helper turns a backend\n # error into False for adopt/reap callers, which would conflate an\n # outage between these two round trips with a peer takeover and\n # evict a live sandbox. Unknown means keep-and-retry here, exactly as\n # it does when the `renew()` call itself cannot answer above.\n try:\n if self._ownership.claim(sandbox_id):\n logger.info(\"Re-established a lapsed ownership lease for %s\", sandbox_id)\n return True\n except OwnershipBackendError as e:\n logger.warning(\"Could not re-establish lapsed lease for %s, will retry: %s\", sandbox_id, e)\n return True\n logger.warning(\"Lapsed ownership lease for %s was taken by a peer\", sandbox_id)\n return False\n return False\n\n @contextlib.contextmanager\n def _held_teardown_lease(self, sandbox_id: str):\n \"\"\"Keep *sandbox_id*'s teardown marker alive for as long as its stop runs.\n\n ``claim(..., for_destroy=True)`` writes the ``del:`` marker with the\n ordinary lease TTL, and normal ``renew()`` extends only ``own:`` while\n deliberately reporting a teardown as ``LOST``. Active and unhealthy\n destroy paths drop the sandbox from the maps ``_renew_owned_leases``\n iterates; the warm path keeps its entry visible until the stop succeeds,\n so ``_forget_lost_sandbox`` separately honours ``_local_teardown`` rather\n than misreading our own marker as a peer takeover. Without this heartbeat,\n a container stop that outlived the TTL let the marker lapse, a peer's\n ``take()`` succeeded against the still-running container, and the stop\n landed on the turn that had just been handed it — the very window the\n ``del:`` state exists to close, reopened by its own expiry.\n\n This is what the per-sandbox ``flock`` used to cover for free: a held lock\n cannot expire. A lease can, so the exclusion has to be held deliberately\n rather than assumed to outlast the work it guards. Reachable without an\n abnormal backend — the config schema bounds only ``renewal_interval_seconds``\n (> 0) and ``ttl_multiplier`` (>= 2), so a legal setting puts the TTL below a\n normal container stop, and ``LocalContainerBackend._stop_container`` passes\n no ``timeout`` to ``subprocess.run``, so a wedged daemon blocks unbounded\n even at the default 120s.\n\n The TTL stays finite on purpose: the heartbeat dies with the process, so a\n destroyer that crashes mid-stop still releases the container one TTL later\n instead of marking it undestroyable forever.\n\n The final release is the heartbeat's own last act, not the caller's. A\n refresh ``claim`` still in flight when the context exits (the socket\n timeout bounds it, but it can be mid-call) would otherwise land *after* a\n caller-side release and rewrite the ``del:`` marker on a container whose\n stop had already completed — stranding a fresh ``take()`` (or rolling back\n a fresh create) until the TTL. Releasing from inside the heartbeat, after\n its loop has stopped, sequences the release strictly after the last\n refresh, so no claim can follow it.\n \"\"\"\n stop = threading.Event()\n\n def beat() -> None:\n interval = self._ownership_config.renewal_interval_seconds\n try:\n while not stop.wait(interval):\n try:\n if not self._ownership.claim(sandbox_id, for_destroy=True):\n # Only reachable if the store lost our marker *and* a\n # peer took it (e.g. a flush mid-stop). The stop is\n # already in flight and cannot be recalled, so say so\n # loudly rather than let a peer's container die without\n # a trace.\n logger.error(\n \"Lost the teardown exclusion for %s while its container stop was still in flight; a peer may have taken it\",\n sandbox_id,\n )\n return\n except Exception as e:\n # Broad on purpose: a refresh that raises must not kill the\n # heartbeat and strand the marker for a stop that can run\n # unbounded. Unknown, not lost — the marker may still be\n # live and the TTL bounds a stale one. Retry on the next tick.\n logger.warning(\"Could not refresh the teardown lease for %s, will retry: %s\", sandbox_id, e)\n finally:\n # Release last, from the heartbeat itself, so an in-flight refresh\n # can never run after the marker is cleared. `release()` drops only\n # our own lease, so this is a safe no-op if a peer took it above.\n self._release_ownership(sandbox_id)\n\n beater = threading.Thread(target=beat, name=\"sandbox-teardown-lease\", daemon=True)\n beater.start()\n try:\n yield\n finally:\n stop.set()\n beater.join(timeout=self._TEARDOWN_JOIN_TIMEOUT_SECONDS)\n if beater.is_alive():\n # The budget covers a normally timing-out refresh plus the final\n # release. The release is the heartbeat's job and is still\n # pending; clearing the marker here would reopen the exact race\n # this owns, so leave it — the thread will release when it\n # unblocks, or the TTL will reap it.\n logger.warning(\n \"Teardown heartbeat for %s did not exit within %.1fs; its lease release is deferred to that thread\",\n sandbox_id,\n self._TEARDOWN_JOIN_TIMEOUT_SECONDS,\n )\n\n # ── Startup reconciliation ────────────────────────────────────────────\n\n def _adoptable_after_grace(self, sandbox_id: str, now: float) -> bool:\n \"\"\"Whether *sandbox_id* has looked unowned long enough to be a real orphan.\n\n An absent lease normally proves the owner died and its TTL ran out. But\n the store can lose every key while every owner is alive and serving — a\n Redis restart without persistence, or eviction under ``maxmemory``\n pressure. ``_refresh_ownership`` already refuses to read that as\n abandonment (``LAPSED`` is re-established, not surrendered). Reading the\n same signal as \"orphan, adopt\" here would contradict it on the other\n path: whoever reconciles first would adopt every live container in the\n window before its owner's next renewal tick, that owner's renewal would\n then report ``LOST``, and it would drop a sandbox it is actively serving\n for the adopter to idle-destroy — #4206 through the back door.\n\n Waiting one full lease TTL rebuilds the delay the state loss erased. A\n live owner republishes within one renewal interval, which is shorter than\n the TTL by construction (``ttl_multiplier >= 2``), so only a container\n whose owner is really gone stays unowned across the whole grace.\n \"\"\"\n if not self._ownership.supports_cross_process:\n # No peer can hold a lease this store would show us, so an unowned\n # container cannot be a live peer's — it is from a dead lifecycle of\n # this process. Single-instance deployments keep instant cleanup, and\n # a grace could not help a multi-worker one on this store anyway:\n # peers are invisible to each other's leases with or without it.\n return True\n\n try:\n current_owner = self._ownership.owner(sandbox_id)\n except OwnershipBackendError as e:\n # Unknown, not free: fail closed, same as _claim_ownership.\n logger.warning(\"Could not read sandbox ownership for %s during reconciliation (deferring adoption): %s\", sandbox_id, e)\n return False\n\n if current_owner is not None:\n # Owned — by a peer, or already by us. Either way not an orphan, and\n # a live owner republishing must restart the grace rather than let a\n # stale one expire over its lease.\n self._unowned_since.pop(sandbox_id, None)\n return False\n\n first_seen = self._unowned_since.setdefault(sandbox_id, now)\n return now - first_seen >= compute_lease_ttl(self._ownership_config)\n\n def _reconcile_orphans(self) -> None:\n \"\"\"Reconcile orphaned containers left by previous process lifecycles.\n\n On startup (and periodically from the idle checker), enumerate running\n containers matching our prefix and adopt **true orphans** into the warm\n pool. A container is only adopted when this instance can claim its\n ownership lease — so multi-instance gateways cannot adopt and later\n idle-destroy a peer's live sandbox (#4206).\n\n Adopted orphans get a fresh warm-pool timestamp; the idle checker then\n destroys them if nobody re-acquires within ``idle_timeout``. That still\n cleans containers left by a crashed process once its lease expires.\n\n An unowned container is not adopted on sight — it must stay unowned for a\n recovery grace first, so a store that lost its state cannot be mistaken\n for a fleet of dead owners (see ``_adoptable_after_grace``).\n \"\"\"\n try:\n running = self._backend.list_running()\n except Exception as e:\n logger.warning(f\"Failed to enumerate running containers during startup reconciliation: {e}\")\n return\n\n # Forget grace timers for containers that no longer exist, so a\n # long-lived instance does not accumulate an entry per destroyed\n # container. Runs before the empty-list return so it also drains.\n running_ids = {info.sandbox_id for info in running}\n self._unowned_since = {sid: seen for sid, seen in self._unowned_since.items() if sid in running_ids}\n\n if not running:\n return\n\n current_time = time.time()\n adopted = 0\n skipped_live = 0\n deferred = 0\n\n for info in running:\n age = current_time - info.created_at if info.created_at > 0 else float(\"inf\")\n if not self._adoptable_after_grace(info.sandbox_id, current_time):\n deferred += 1\n logger.debug(\"Deferring container %s during reconciliation: owned, or not yet past the recovery grace\", info.sandbox_id)\n continue\n\n # Claim second: a successful claim proves the container is not a\n # peer's and locks peers out. It says nothing about *us* — it\n # succeeds against our own lease by design — so it is not a substitute\n # for the local teardown check below. The grace above is likewise a\n # precondition, not a substitute; only the claim is atomic.\n if not self._claim_ownership(info.sandbox_id):\n skipped_live += 1\n logger.debug(\"Skipping container %s during reconciliation: owned by another instance\", info.sandbox_id)\n continue\n\n # Single lock acquisition per container: atomic check-and-insert.\n # Avoids a TOCTOU window between the \"already tracked?\" check and the\n # warm-pool insert.\n with self._lock:\n if info.sandbox_id in self._sandboxes or info.sandbox_id in self._warm_pool:\n continue\n if self._being_torn_down_locally(info.sandbox_id):\n # Adoption is a promote, so it needs the same reservation\n # check as the other three. A container being torn down here\n # is untracked and still running, which is exactly the shape\n # this loop adopts — and neither the claim nor the grace\n # excludes it. On `memory` the grace is skipped outright\n # (`supports_cross_process = False`), so nothing else stands\n # in the way there at all; adopting would park a container\n # into the warm pool moments before its stop lands, leaving a\n # dead entry for the next reclaim to hand out.\n deferred += 1\n logger.debug(\"Deferring container %s during reconciliation: this instance is tearing it down\", info.sandbox_id)\n continue\n self._warm_pool[info.sandbox_id] = (info, current_time)\n self._warm_pool_identity[info.sandbox_id] = None\n self._unowned_since.pop(info.sandbox_id, None)\n adopted += 1\n logger.info(f\"Adopted container {info.sandbox_id} into warm pool (age: {age:.0f}s)\")\n\n logger.info(\n \"Startup reconciliation complete: %s adopted into warm pool, %s skipped (live peer ownership), %s deferred (owned or within recovery grace), %s total found\",\n adopted,\n skipped_live,\n deferred,\n len(running),\n )\n\n # ── Deterministic ID ─────────────────────────────────────────────────\n\n @staticmethod\n def _effective_acquire_user_id(user_id: str | None) -> str:\n return user_id or get_effective_user_id()\n\n @staticmethod\n def _thread_key(thread_id: str, user_id: str) -> tuple[str, str]:\n return (user_id, thread_id)\n\n @staticmethod\n def _deterministic_sandbox_id(thread_id: str, user_id: str) -> str:\n \"\"\"Generate a deterministic sandbox ID from user/thread scope.\n\n Includes user_id so a previously-created default-bucket sandbox cannot be\n reused for an auth/channel run that should mount a user-scoped bucket.\n\n During a mixed-version rollout, older 8-character containers are not\n reused under the new 16-character identity. They remain eligible for\n normal orphan cleanup while the first new-version acquire cold-starts.\n \"\"\"\n return hashlib.sha256(f\"{user_id}:{thread_id}\".encode()).hexdigest()[:16]\n\n def _assert_active_identity_available_locked(\n self,\n sandbox_id: str,\n requested_key: tuple[str, str],\n ) -> None:\n \"\"\"Fail closed if an active truncated ID belongs to another identity.\"\"\"\n if sandbox_id not in self._sandboxes and sandbox_id not in self._sandbox_infos:\n return\n\n stored_key = self._active_sandbox_identity.get(sandbox_id)\n if stored_key is None:\n matching_keys = [key for key, mapped_id in self._thread_sandboxes.items() if mapped_id == sandbox_id]\n if len(matching_keys) == 1:\n stored_key = matching_keys[0]\n if stored_key != requested_key:\n raise SandboxIdentityCollisionError(sandbox_id, stored_key, requested_key)\n\n def _assert_warm_identity_available_locked(\n self,\n sandbox_id: str,\n requested_key: tuple[str, str],\n ) -> None:\n \"\"\"Fail closed if a warm ID changed tenants during an acquire.\"\"\"\n if sandbox_id not in self._warm_pool:\n return\n # Startup-adopted entries have unknown identity until their first reclaim.\n stored_key = self._warm_pool_identity.get(sandbox_id)\n if stored_key is not None and stored_key != requested_key:\n raise SandboxIdentityCollisionError(sandbox_id, stored_key, requested_key)\n\n # ── Mount helpers ────────────────────────────────────────────────────\n\n def _get_extra_mounts(self, thread_id: str | None, *, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Collect all extra mounts for a sandbox (thread-specific + skills).\"\"\"\n mounts: list[tuple[str, str, bool]] = []\n\n if thread_id:\n mounts.extend(self._get_thread_mounts(thread_id, user_id=user_id))\n logger.info(f\"Adding thread mounts for thread {thread_id}: {mounts}\")\n\n skills_mounts = self._get_skills_mounts(user_id=user_id)\n if skills_mounts:\n mounts.extend(skills_mounts)\n logger.info(f\"Adding skills mounts: {skills_mounts}\")\n\n user_skill_mounts = self._get_user_skill_mounts(user_id=user_id)\n if user_skill_mounts:\n mounts.extend(user_skill_mounts)\n logger.info(f\"Adding user skill mounts: {user_skill_mounts}\")\n\n lark_cli_mounts = self._get_lark_cli_runtime_mounts(user_id=user_id)\n if lark_cli_mounts:\n mounts.extend(lark_cli_mounts)\n logger.info(f\"Adding Lark CLI runtime mounts: {lark_cli_mounts}\")\n\n return self._dedupe_mounts_by_container_path(mounts)\n\n @staticmethod\n def _dedupe_mounts_by_container_path(mounts: list[tuple[str, str, bool]]) -> list[tuple[str, str, bool]]:\n \"\"\"Keep the first mount for each container path.\n\n Duplicate container paths are rejected by the provisioner and can also\n fail local Docker creation. The earlier mount wins because mount helpers\n are appended in priority order: thread data, skill roots, integration\n skill roots, then integration runtimes/credentials.\n \"\"\"\n seen: set[str] = set()\n deduped: list[tuple[str, str, bool]] = []\n for host_path, container_path, read_only in mounts:\n if container_path in seen:\n logger.warning(\n \"Skipping duplicate sandbox mount for container path %s from host %s\",\n container_path,\n host_path,\n )\n continue\n seen.add(container_path)\n deduped.append((host_path, container_path, read_only))\n return deduped\n\n @staticmethod\n def _get_thread_mounts(thread_id: str, *, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Get volume mounts for a thread's data directories.\n\n Creates directories if they don't exist (lazy initialization).\n Mount sources use host_base_dir so that when running inside Docker with a\n mounted Docker socket (DooD), the host Docker daemon can resolve the paths.\n \"\"\"\n paths = get_paths()\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n paths.ensure_thread_dirs(thread_id, user_id=effective_user_id)\n\n return [\n (paths.host_sandbox_work_dir(thread_id, user_id=effective_user_id), f\"{VIRTUAL_PATH_PREFIX}/workspace\", False),\n (paths.host_sandbox_uploads_dir(thread_id, user_id=effective_user_id), f\"{VIRTUAL_PATH_PREFIX}/uploads\", False),\n (paths.host_sandbox_outputs_dir(thread_id, user_id=effective_user_id), f\"{VIRTUAL_PATH_PREFIX}/outputs\", False),\n # ACP workspace: read-only inside the sandbox (lead agent reads results;\n # the ACP subprocess writes from the host side, not from within the container).\n (paths.host_acp_workspace_dir(thread_id, user_id=effective_user_id), \"/mnt/acp-workspace\", True),\n ]\n\n @staticmethod\n def _get_skills_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Get skills directory mount configurations for three-way skills layout.\n\n Mirrors ``LocalSandboxProvider._build_thread_path_mappings`` for AIO\n sandboxes: public, per-user custom, and legacy (pre-migration\n global-custom) skills are mounted to separate container subdirectories so\n that ``Skill.get_container_path()`` category-aware paths resolve\n correctly inside the sandbox.\n\n Mount sources use ``DEER_FLOW_HOST_BASE_DIR`` when running inside\n Docker (DooD) so the host Docker daemon can resolve the projection\n paths.\n \"\"\"\n mounts: list[tuple[str, str, bool]] = []\n try:\n config = get_app_config()\n container_path = config.skills.container_path\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n AioSandboxProvider._ensure_skills_projection(effective_user_id)\n paths = get_paths()\n host_base_dir = str(paths.host_base_dir)\n\n # 1. Public skills: global, read-only — static, shared by all threads\n mounts.append(\n (\n join_host_path(host_base_dir, \"skills_view\", \"public\"),\n f\"{container_path}/public\",\n True,\n )\n )\n\n # 2. Per-user custom skills: read-only, per-thread/per-user\n host_user_custom = join_host_path(\n host_base_dir,\n \"users\",\n effective_user_id,\n \"skills_view\",\n \"custom\",\n )\n mounts.append(\n (\n host_user_custom,\n f\"{container_path}/custom\",\n True,\n )\n )\n\n # 3. Legacy visibility is encoded by projection contents. Keep the\n # mount stable even when the directory is empty so a later state\n # change is visible without recreating the sandbox.\n mounts.append(\n (\n join_host_path(host_base_dir, \"users\", effective_user_id, \"skills_view\", \"legacy\"),\n f\"{container_path}/legacy\",\n True,\n )\n )\n except Exception as e:\n logger.warning(\"Could not setup skills mounts: %s\", e)\n\n return mounts\n\n @staticmethod\n def _ensure_skills_projection(user_id: str):\n \"\"\"Best-effort: a projection failure must not fail sandbox acquire.\n\n Called directly (for its side effect) from ``_acquire_internal`` /\n ``_acquire_internal_async`` outside any try/except, as well as from\n within ``_get_skills_mounts``'s own guarded block — swallowing here\n keeps both call sites safe without duplicating the guard.\n \"\"\"\n from deerflow.skills.projection import ensure_skill_projections\n from deerflow.skills.storage import get_or_new_user_skill_storage\n\n try:\n storage = get_or_new_user_skill_storage(user_id, app_config=get_app_config())\n return ensure_skill_projections(storage)\n except Exception as exc:\n logger.warning(\"Could not ensure skills projection for user %s: %s\", user_id, exc, exc_info=True)\n return None\n\n @staticmethod\n def _get_user_skill_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Mount enabled managed integration skills into AIO sandboxes.\n\n Per-user custom skills are already mounted by ``_get_skills_mounts``.\n Integration packages are shared, but their enabled state is per-user, so\n this helper mounts the user's projection instead of the raw shared root.\n \"\"\"\n try:\n config = get_app_config()\n paths = get_paths()\n skills_container_path = config.skills.container_path\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n AioSandboxProvider._ensure_skills_projection(effective_user_id)\n return [\n (\n join_host_path(\n str(paths.host_base_dir),\n \"users\",\n effective_user_id,\n \"skills_view\",\n \"integrations\",\n ),\n f\"{skills_container_path}/integrations\",\n True,\n ),\n ]\n except Exception as e:\n logger.warning(f\"Could not setup user skill mounts: {e}\")\n return []\n\n @staticmethod\n def _lark_integration_active(user_id: str | None = None) -> bool:\n \"\"\"Whether the managed Lark skill pack is installed for this user.\n\n Drives whether a sandbox requests the lark-cli runtime (init container /\n Gateway-download mount). Independent of whether a local ``sandbox-cli``\n dir exists, so remote/K8s can opt in without a Gateway-side download.\n \"\"\"\n try:\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n return lark_skills_installed(effective_user_id)\n except Exception as e: # pragma: no cover - defensive\n logger.warning(f\"Could not determine Lark integration state: {e}\")\n return False\n\n @staticmethod\n def _lark_broker_active(user_id: str | None = None) -> bool:\n \"\"\"Whether this user's sandbox should use the lark-cli broker (Pattern B).\n\n True only when the Lark pack is installed AND the remote provisioner\n reports a configured broker image. When true, the provisioner keeps the\n credentials in a sidecar and the sandbox gets only a shim, so the\n Gateway-side credential-mount overlay must not run either.\n \"\"\"\n try:\n if not AioSandboxProvider._lark_integration_active(user_id):\n return False\n from deerflow.integrations.lark_cli import sandbox_lark_broker_active\n\n return sandbox_lark_broker_active()\n except Exception as e: # pragma: no cover - defensive\n logger.warning(f\"Could not determine Lark broker state: {e}\")\n return False\n\n @staticmethod\n def _get_lark_cli_runtime_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Mount the per-user lark-cli config/data dirs used by Settings auth.\n\n Settings endpoints run ``lark-cli`` on the Gateway with\n ``LARKSUITE_CLI_CONFIG_DIR`` / ``DATA_DIR`` pointing at\n ``users/{user}/integrations/lark-cli``. Agent conversations run\n ``lark-cli`` inside the sandbox, so those same directories must be\n mounted into the container or the CLI sees a separate unauthenticated\n profile.\n\n The ``config`` dir holds the long-lived Lark ``appSecret`` (written by\n ``lark-cli config init`` on the Gateway, never in-sandbox), so it is\n mounted **read-only**: sandbox processes only need to read it, and a\n read-only bind stops a compromised agent from tampering with or\n replacing the app credentials. The ``data`` dir holds refreshable OAuth\n tokens that ``lark-cli auth`` updates in-sandbox, so it stays writable.\n This is defense-in-depth only — both dirs remain readable to arbitrary\n sandbox processes until the auth-proxy follow-up (issue #4338) lands.\n See the sandbox trust-boundary note in ``backend/AGENTS.md``.\n \"\"\"\n try:\n paths = get_paths()\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n ensure_lark_cli_credential_tree(effective_user_id, paths=paths)\n mounts = [\n (paths.host_user_integration_config_dir(effective_user_id, LARK_CLI_INTEGRATION_ID), LARK_CLI_SANDBOX_CONFIG_DIR, True),\n (paths.host_user_integration_data_dir(effective_user_id, LARK_CLI_INTEGRATION_ID), LARK_CLI_SANDBOX_DATA_DIR, False),\n ]\n runtime_dir = paths.base_dir / \"integrations\" / LARK_CLI_INTEGRATION_ID / \"sandbox-cli\"\n if runtime_dir.is_dir():\n mounts.append(\n (\n join_host_path(str(paths.host_base_dir), \"integrations\", LARK_CLI_INTEGRATION_ID, \"sandbox-cli\"),\n LARK_CLI_SANDBOX_RUNTIME_DIR,\n True,\n )\n )\n return mounts\n except Exception as e:\n logger.warning(f\"Could not setup Lark CLI runtime mounts: {e}\")\n return []\n\n # ── Idle timeout management ──────────────────────────────────────────\n\n def _cleanup_idle_resources(self, idle_timeout: float) -> None:\n \"\"\"Clean AIO resources idle longer than ``idle_timeout`` seconds.\"\"\"\n # Pick up containers whose peer leases expired since startup (crash path).\n self._reconcile_orphans()\n self._cleanup_idle_sandboxes(idle_timeout)\n\n # ── Ownership lease renewal ──────────────────────────────────────────\n\n def _start_lease_renewal(self) -> None:\n \"\"\"Start the daemon thread that keeps this instance's leases alive.\n\n Deliberately not folded into the idle checker: that thread only starts\n when ``idle_timeout > 0``, so renewal riding on it silently stopped for\n ``idle_timeout: 0`` deployments — a supported config (\"keep warm VMs\n until shutdown\") — letting every lease lapse and reopening #4206 one TTL\n later. Liveness and reaping must not share a switch.\n \"\"\"\n if self._renewal_thread is not None and self._renewal_thread.is_alive():\n return\n\n self._renewal_stop.clear()\n self._renewal_thread = threading.Thread(\n target=self._lease_renewal_loop,\n name=\"sandbox-lease-renewal\",\n daemon=True,\n )\n self._renewal_thread.start()\n logger.info(\n \"Started sandbox ownership renewal thread (interval: %.1fs, ttl: %.1fs)\",\n self._ownership_config.renewal_interval_seconds,\n self._ownership_config.renewal_interval_seconds * self._ownership_config.ttl_multiplier,\n )\n\n def _stop_lease_renewal(self) -> None:\n self._renewal_stop.set()\n thread = self._renewal_thread\n if thread is not None and thread.is_alive() and thread is not threading.current_thread():\n thread.join(timeout=5)\n\n def _lease_renewal_loop(self) -> None:\n interval = self._ownership_config.renewal_interval_seconds\n while not self._renewal_stop.wait(interval):\n try:\n self._renew_owned_leases()\n except Exception:\n logger.exception(\"Error in sandbox ownership renewal loop\")\n\n def _renew_owned_leases(self) -> None:\n \"\"\"Renew every container this instance believes it owns.\n\n Covers warm entries as well as active ones: a warm container is still\n ours (we hold it for fast reclaim), so letting its lease lapse would let\n a peer adopt a container we are about to hand back to its thread.\n\n Only a lease a **peer** now holds means the container is no longer ours;\n a lapsed one is re-established (see ``_refresh_ownership``). Conflating\n the two would evict every live sandbox on this instance the first time\n the store lost its state.\n \"\"\"\n with self._lock:\n owned_ids = list(self._sandboxes.keys()) + list(self._warm_pool.keys())\n\n for sandbox_id in owned_ids:\n # Snapshot before the round trip: by the time `renew()` answers LOST,\n # an acquire in this process may already have taken the lease back\n # and promoted the id, and the answer is about the lease we held then.\n epoch = self._acquire_epoch_of(sandbox_id)\n if not self._refresh_ownership(sandbox_id):\n logger.warning(\"Lost sandbox ownership lease for %s; dropping it from this instance\", sandbox_id)\n self._forget_lost_sandbox(sandbox_id, expected_epoch=epoch)\n\n def _forget_lost_sandbox(self, sandbox_id: str, *, expected_epoch: int | None = None) -> None:\n \"\"\"Drop a sandbox whose lease we no longer hold, without touching the container.\n\n The container now belongs to whichever instance holds the lease, so\n stopping it here would be the very cross-instance kill this store exists\n to prevent. Only our host-side handle goes away.\n\n ``expected_epoch`` guards callers whose \"we lost it\" decision came from a\n store round trip made outside the lock. An acquire **mid-flight** counts\n too: its ``take()`` can already have made the takeover durable while the\n epoch is still unwritten, so the epoch alone would let a stale decision\n through (see ``_publish_ownership``). An acquire that re-took the lease\n in that window has already handed the sandbox to a turn — and, on the\n reuse path, handed out the *same* tracked client, so no object-identity\n check would notice. Dropping it then closes a client mid-turn and leaves\n the agent holding an id whose tool calls fail until the next turn.\n \"\"\"\n with self._lock:\n # A warm teardown deliberately keeps its entry visible until the\n # backend stop succeeds. Its own `del:` marker makes `renew()` report\n # LOST, but that is not a peer takeover and must not pop the retained\n # entry — especially when the stop fails and the container remains\n # live for retry/reclaim. The teardown path removes it on success.\n if sandbox_id in self._local_teardown:\n logger.debug(\"Not dropping sandbox %s: this instance is tearing it down\", sandbox_id)\n return\n # The in-flight check is deliberately *not* conditional on\n # `expected_epoch`. Today's epoch-less callers (the two\n # `SandboxBeingDestroyedError` handlers) cannot collide with a\n # publish for the same id — `_publish_ownership` has already cleared\n # the mark by the time they run, and acquires for one id are\n # serialized by the per-thread lock — so this changes no current\n # behaviour. It is here because \"no epoch supplied\" reading as \"no\n # guard at all\" is how the next caller of a dangerous primitive gets\n # written; an id being acquired right now must never be dropped.\n if sandbox_id in self._acquire_inflight:\n logger.info(\"Not dropping sandbox %s: an acquire is publishing ownership for it\", sandbox_id)\n return\n if expected_epoch is not None and self._acquire_epoch.get(sandbox_id, 0) != expected_epoch:\n logger.info(\"Not dropping sandbox %s: this instance re-acquired it after the lease check\", sandbox_id)\n return\n\n sandbox = self._sandboxes.pop(sandbox_id, None)\n self._sandbox_infos.pop(sandbox_id, None)\n self._active_sandbox_identity.pop(sandbox_id, None)\n self._last_activity.pop(sandbox_id, None)\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n self._acquire_epoch.pop(sandbox_id, None)\n for key, mapped_id in list(self._thread_sandboxes.items()):\n if mapped_id == sandbox_id:\n del self._thread_sandboxes[key]\n\n # Close the host-side HTTP client we are dropping (#2872); the container\n # itself stays up for its new owner.\n if sandbox is not None:\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} after losing its lease: {e}\")\n\n def _cleanup_idle_sandboxes(self, idle_timeout: float) -> None:\n current_time = time.time()\n active_to_destroy = []\n\n with self._lock:\n # Active sandboxes: tracked via _last_activity\n for sandbox_id, last_activity in self._last_activity.items():\n idle_duration = current_time - last_activity\n if idle_duration > idle_timeout:\n active_to_destroy.append(sandbox_id)\n logger.info(f\"Sandbox {sandbox_id} idle for {idle_duration:.1f}s, marking for destroy\")\n\n # Destroy active sandboxes (re-verify still idle before acting).\n #\n # The re-verify has to happen in the same critical section as the\n # teardown reservation, which is why it is handed to `_destroy_tracked`\n # as a predicate rather than run here. Checking here and destroying\n # afterwards left a window — widened by this PR from a few instructions\n # to a store round trip, since `destroy()` now claims ownership before it\n # untracks — in which a turn re-acquires the sandbox and then has its\n # container stopped underneath it.\n def still_idle(sandbox_id: str) -> bool:\n last_activity = self._last_activity.get(sandbox_id)\n if last_activity is None:\n # Already released or destroyed by another path — skip.\n logger.info(f\"Sandbox {sandbox_id} already gone before idle destroy, skipping\")\n return False\n if (time.time() - last_activity) < idle_timeout:\n # Re-acquired (activity updated) since the snapshot — skip.\n logger.info(f\"Sandbox {sandbox_id} was re-acquired before idle destroy, skipping\")\n return False\n return True\n\n for sandbox_id in active_to_destroy:\n try:\n logger.info(f\"Destroying idle sandbox {sandbox_id}\")\n self._destroy_tracked(sandbox_id, still_reapable=lambda sid=sandbox_id: still_idle(sid))\n except Exception as e:\n logger.error(f\"Failed to destroy idle sandbox {sandbox_id}: {e}\")\n\n self._reap_expired_warm(idle_timeout)\n\n def _reap_expired_warm(self, idle_timeout: float | None = None) -> None:\n \"\"\"Destroy warm entries older than ``idle_timeout``, never a peer's live container.\"\"\"\n timeout = float(self._config.get(\"idle_timeout\", DEFAULT_IDLE_TIMEOUT) if idle_timeout is None else idle_timeout)\n if timeout <= 0:\n return\n\n now = time.time()\n expired: list[tuple[str, SandboxInfo]] = []\n with self._lock:\n for sandbox_id, (entry, timestamp) in self._warm_pool.items():\n if now - timestamp > timeout:\n expired.append((sandbox_id, entry))\n\n # Only drop an entry from the warm pool once we know it is really going\n # away. Popping first would lose the container on a refused or\n # unanswerable claim: still running, no longer tracked by anyone. The\n # deferred pop is why the reservation is needed — the entry stays visible\n # to `_reclaim_warm_pool_sandbox` for the whole stop.\n for sandbox_id, entry in expired:\n self._destroy_warm_entry(sandbox_id, entry, reason=\"idle_timeout\", still_reapable=lambda sid=sandbox_id: sid in self._warm_pool)\n\n def _evict_oldest_warm(self) -> str | None:\n \"\"\"Evict the oldest warm entry this instance still owns.\"\"\"\n with self._lock:\n if not self._warm_pool:\n return None\n # Snapshot oldest-first under the lock; ownership is resolved outside\n # it, since a claim can be a network round trip and the provider lock\n # guards every acquire path.\n candidates = [(sandbox_id, entry) for sandbox_id, (entry, _) in sorted(self._warm_pool.items(), key=lambda item: item[1][1])]\n\n for sandbox_id, entry in candidates:\n # \"Still in the warm pool?\" is the reapable check, and it has to run\n # in the same critical section as the reservation — checking it here\n # and reserving afterwards is exactly the window a reclaim slips\n # through. `_destroy_warm_entry` does both under one lock hold.\n if not self._destroy_warm_entry(sandbox_id, entry, reason=\"replica_enforcement\", still_reapable=lambda sid=sandbox_id: sid in self._warm_pool):\n continue\n return sandbox_id\n\n return None\n\n # ── Signal handling ──────────────────────────────────────────────────\n\n def _register_signal_handlers(self) -> None:\n \"\"\"Register signal handlers for graceful shutdown.\n\n Handles SIGTERM, SIGINT, and SIGHUP (terminal close) to ensure\n sandbox containers are cleaned up even when the user closes the terminal.\n \"\"\"\n self._original_sigterm = signal.getsignal(signal.SIGTERM)\n self._original_sigint = signal.getsignal(signal.SIGINT)\n self._original_sighup = signal.getsignal(signal.SIGHUP) if hasattr(signal, \"SIGHUP\") else None\n\n def signal_handler(signum, frame):\n self.shutdown()\n if signum == signal.SIGTERM:\n original = self._original_sigterm\n elif hasattr(signal, \"SIGHUP\") and signum == signal.SIGHUP:\n original = self._original_sighup\n else:\n original = self._original_sigint\n if callable(original):\n original(signum, frame)\n elif original == signal.SIG_DFL:\n signal.signal(signum, signal.SIG_DFL)\n signal.raise_signal(signum)\n\n try:\n signal.signal(signal.SIGTERM, signal_handler)\n signal.signal(signal.SIGINT, signal_handler)\n if hasattr(signal, \"SIGHUP\"):\n signal.signal(signal.SIGHUP, signal_handler)\n except ValueError:\n logger.debug(\"Could not register signal handlers (not main thread)\")\n\n # ── Thread locking (in-process) ──────────────────────────────────────\n\n def _get_thread_lock(self, thread_id: str, user_id: str) -> threading.Lock:\n \"\"\"Get or create an in-process lock for a specific user/thread scope.\"\"\"\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n if key not in self._thread_locks:\n self._thread_locks[key] = threading.Lock()\n return self._thread_locks[key]\n\n def _sandbox_id_for_thread(self, thread_id: str | None, user_id: str | None) -> str:\n \"\"\"Return deterministic IDs for thread sandboxes and random IDs otherwise.\"\"\"\n return self._deterministic_sandbox_id(thread_id, self._effective_acquire_user_id(user_id)) if thread_id else str(uuid.uuid4())[:8]\n\n def _reuse_in_process_sandbox(self, thread_id: str | None, *, user_id: str | None = None, post_lock: bool = False) -> str | None:\n \"\"\"Reuse an active in-process sandbox for a thread if one is still tracked.\"\"\"\n if thread_id is None:\n return None\n\n effective_user_id = self._effective_acquire_user_id(user_id)\n key = self._thread_key(thread_id, effective_user_id)\n with self._lock:\n if key not in self._thread_sandboxes:\n return None\n\n existing_id = self._thread_sandboxes[key]\n if self._being_torn_down_locally(existing_id):\n # A reaper thread in this process is stopping this container.\n # Same answer as a peer's `del:` lease: cold-start instead.\n logger.info(\"Cached sandbox %s is being destroyed by this instance; not reusing it\", existing_id)\n return None\n if existing_id in self._sandboxes:\n info = self._sandbox_infos.get(existing_id)\n else:\n del self._thread_sandboxes[key]\n return None\n\n alive = self._check_tracked_sandbox_alive(existing_id, info) if info is not None else True\n if alive is False:\n self._drop_unhealthy_sandbox(\n existing_id,\n \"in-process cache failed health check\",\n expected_info=info,\n )\n return None\n\n with self._lock:\n if self._thread_sandboxes.get(key) != existing_id:\n return None\n if existing_id not in self._sandboxes:\n self._thread_sandboxes.pop(key, None)\n return None\n\n suffix = \" (post-lock check)\" if post_lock else \"\"\n logger.info(f\"Reusing in-process sandbox {existing_id} for user/thread {effective_user_id}/{thread_id}{suffix}\")\n self._last_activity[existing_id] = time.time()\n\n # Fail closed: an OwnershipBackendError propagates rather than handing out\n # a sandbox we could not publish ownership for.\n try:\n self._publish_ownership(existing_id)\n except SandboxBeingDestroyedError:\n # A peer is stopping this container. Drop it and let the caller\n # discover-or-create a fresh one instead of handing over a sandbox\n # that is about to disappear.\n logger.info(\"Cached sandbox %s is being destroyed by another instance; not reusing it\", existing_id)\n self._forget_lost_sandbox(existing_id)\n return None\n\n with self._lock:\n if self._being_torn_down_locally(existing_id):\n # The first reservation check ran before the backend health\n # check and ownership round trip. A local reaper can win while\n # either is in flight, and it deliberately keeps the entry in\n # `_sandboxes` until its destroy claim succeeds. Membership\n # alone therefore cannot prove this id is still safe to return.\n logger.info(\"Cached sandbox %s was reserved for teardown while publishing ownership; not reusing it\", existing_id)\n return None\n if existing_id not in self._sandboxes:\n # Dropped while we were publishing. The intent mark closes the\n # window *inside* `_publish_ownership`, but not the gap before\n # it: until the mark is set a renewal's `LOST` is both current\n # and correct — the peer really did hold the lease — so the\n # forget legitimately runs and closes this client. Returning the\n # id anyway would hand back a sandbox whose `get()` is `None`.\n # Fall through instead; the caller re-discovers and builds a\n # fresh client, and the lease we just took is already ours.\n logger.info(\"Cached sandbox %s was dropped while publishing ownership; falling through to discovery\", existing_id)\n return None\n return existing_id\n\n def _reclaim_warm_pool_sandbox(\n self,\n thread_id: str | None,\n sandbox_id: str,\n *,\n user_id: str | None = None,\n post_lock: bool = False,\n ) -> str | None:\n \"\"\"Promote a warm-pool sandbox back to active tracking if available.\"\"\"\n if thread_id is None:\n return None\n\n effective_user_id = self._effective_acquire_user_id(user_id)\n key = self._thread_key(thread_id, effective_user_id)\n with self._lock:\n if sandbox_id not in self._warm_pool:\n return None\n self._assert_warm_identity_available_locked(sandbox_id, key)\n if self._being_torn_down_locally(sandbox_id):\n # The entry deliberately stays in `_warm_pool` for the whole stop\n # (so a refused claim does not lose the container), so pool\n # membership alone does not mean it is reclaimable.\n logger.info(\"Warm-pool sandbox %s is being destroyed by this instance; not reclaiming it\", sandbox_id)\n return None\n\n info, _ = self._warm_pool[sandbox_id]\n\n alive = self._check_tracked_sandbox_alive(sandbox_id, info)\n if alive is False:\n self._drop_unhealthy_sandbox(\n sandbox_id,\n \"warm-pool cache failed health check\",\n expected_info=info,\n )\n return None\n\n # Publish ownership before the warm → active transition: a raise here must\n # not leave the sandbox tracked as active but unowned (a peer would see an\n # orphan and reap it mid-turn). On failure the entry stays warm and this\n # instance keeps its existing lease.\n try:\n self._publish_ownership(sandbox_id)\n except SandboxBeingDestroyedError:\n logger.info(\"Warm-pool sandbox %s is being destroyed by another instance; not reclaiming it\", sandbox_id)\n self._forget_lost_sandbox(sandbox_id)\n return None\n\n with self._lock:\n if self._being_torn_down_locally(sandbox_id):\n # Re-checked, because the first check was before the round trip.\n # A reaper can reserve *after* our `take()` — the warm entry is\n # still there, since its pop is deferred until the stop returns —\n # then claim `del:` (which succeeds: the lease is ours, we just\n # took it) and stop the container. Whichever pop lands first\n # decides, and if ours does we install a client for a container\n # that is already stopped.\n logger.info(\"Warm-pool sandbox %s was claimed for teardown while publishing ownership; not reclaiming it\", sandbox_id)\n return None\n self._assert_warm_identity_available_locked(sandbox_id, key)\n warm_item = self._warm_pool.pop(sandbox_id, None)\n if warm_item is None:\n return None\n self._warm_pool_identity.pop(sandbox_id, None)\n info, _ = warm_item\n sandbox = AioSandbox(id=sandbox_id, base_url=info.sandbox_url)\n self._sandboxes[sandbox_id] = sandbox\n self._sandbox_infos[sandbox_id] = info\n self._active_sandbox_identity[sandbox_id] = key\n self._last_activity[sandbox_id] = time.time()\n self._thread_sandboxes[key] = sandbox_id\n\n suffix = \" (post-lock check)\" if post_lock else f\" at {info.sandbox_url}\"\n logger.info(f\"Reclaimed warm-pool sandbox {sandbox_id} for user/thread {effective_user_id}/{thread_id}{suffix}\")\n return sandbox_id\n\n def _recheck_cached_sandbox(self, thread_id: str, sandbox_id: str, *, user_id: str) -> str | None:\n \"\"\"Re-check in-memory caches after acquiring the cross-process file lock.\"\"\"\n return self._reuse_in_process_sandbox(thread_id, user_id=user_id, post_lock=True) or self._reclaim_warm_pool_sandbox(\n thread_id,\n sandbox_id,\n user_id=user_id,\n post_lock=True,\n )\n\n def _register_discovered_sandbox(self, thread_id: str, info: SandboxInfo, *, user_id: str) -> str:\n \"\"\"Track a sandbox discovered through the backend.\n\n Raises:\n SandboxBeingDestroyedError: discovery found the container still\n running, but a peer is stopping it. Deliberately propagated\n rather than swallowed: falling through to create would collide\n with the not-yet-removed container name, and handing this one to\n an agent is exactly the mid-turn death (#4206) the store exists to\n prevent. The window is a peer's in-flight container stop, so the\n thread's next turn discovers nothing and cold-starts cleanly.\n \"\"\"\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n if self._being_torn_down_locally(info.sandbox_id):\n # Discovery is the fall-through once the caches miss, so it is\n # also the path a reaper's own untracking opens up. `take()` would\n # only refuse this once the reaper's `del:` claim has landed;\n # until then it succeeds against our own lease.\n raise SandboxBeingDestroyedError(info.sandbox_id)\n self._assert_active_identity_available_locked(info.sandbox_id, key)\n self._assert_warm_identity_available_locked(info.sandbox_id, key)\n\n sandbox = AioSandbox(id=info.sandbox_id, base_url=info.sandbox_url)\n # Ownership first, so a failure cannot leave a tracked-but-unowned sandbox.\n # There is no container to roll back (we did not create it), but the\n # host-side HTTP client constructed above is ours and must not leak —\n # same close-on-failure as `_register_created_sandbox`.\n try:\n self._publish_ownership(info.sandbox_id)\n with self._lock:\n if self._being_torn_down_locally(info.sandbox_id):\n # The pre-publish reservation check is only an early-out: a\n # local reaper can reserve the id during the store round\n # trip. Do not install a client for a container that reaper\n # has already committed to stopping.\n raise SandboxBeingDestroyedError(info.sandbox_id)\n self._assert_active_identity_available_locked(info.sandbox_id, key)\n self._assert_warm_identity_available_locked(info.sandbox_id, key)\n # Active and warm are exclusive states, and only this insert can\n # violate that: a warm entry for the same id is stale the moment\n # the id becomes active. Leaving it there gives the container two\n # reapers — `_reap_expired_warm` judges it by the warm timestamp\n # and never looks at `_last_activity`, so it stops a container an\n # agent is actively using while `_sandboxes` still hands out its\n # client.\n self._warm_pool.pop(info.sandbox_id, None)\n self._warm_pool_identity.pop(info.sandbox_id, None)\n self._sandboxes[info.sandbox_id] = sandbox\n self._sandbox_infos[info.sandbox_id] = info\n self._active_sandbox_identity[info.sandbox_id] = key\n self._last_activity[info.sandbox_id] = time.time()\n self._thread_sandboxes[key] = info.sandbox_id\n except (\n OwnershipBackendError,\n SandboxBeingDestroyedError,\n SandboxIdentityCollisionError,\n ):\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {info.sandbox_id} after failed ownership publish: {e}\")\n raise\n\n logger.info(f\"Discovered existing sandbox {info.sandbox_id} for user/thread {user_id}/{thread_id} at {info.sandbox_url}\")\n return info.sandbox_id\n\n def _register_created_sandbox(self, thread_id: str | None, sandbox_id: str, info: SandboxInfo, *, user_id: str | None = None) -> str:\n \"\"\"Track a newly-created sandbox in the active maps.\"\"\"\n sandbox = AioSandbox(id=sandbox_id, base_url=info.sandbox_url)\n key = (\n self._thread_key(\n thread_id,\n self._effective_acquire_user_id(user_id),\n )\n if thread_id\n else None\n )\n # Ownership first. Unlike the discover path there IS something to roll\n # back: we just started this container, and an unowned running container\n # is exactly what a peer's reconciliation adopts. Leaking it would hand a\n # peer a container this instance is about to use.\n # SandboxBeingDestroyedError is possible even here: a peer that died\n # mid-stop leaves a teardown marker until its TTL lapses. Roll back on\n # both, or the container we just started is leaked.\n try:\n if key is not None:\n with self._lock:\n self._assert_active_identity_available_locked(sandbox_id, key)\n self._assert_warm_identity_available_locked(sandbox_id, key)\n self._publish_ownership(sandbox_id)\n\n with self._lock:\n if key is not None:\n self._assert_active_identity_available_locked(sandbox_id, key)\n self._assert_warm_identity_available_locked(sandbox_id, key)\n # Same exclusivity rule as the discover path.\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n self._sandboxes[sandbox_id] = sandbox\n self._sandbox_infos[sandbox_id] = info\n self._active_sandbox_identity[sandbox_id] = key\n self._last_activity[sandbox_id] = time.time()\n if key is not None:\n self._thread_sandboxes[key] = sandbox_id\n except (\n OwnershipBackendError,\n SandboxBeingDestroyedError,\n SandboxIdentityCollisionError,\n ):\n logger.error(\n \"Could not register new sandbox %s; destroying it rather than leaking an untracked container\",\n sandbox_id,\n )\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} during ownership rollback: {e}\")\n try:\n self._backend.destroy(info)\n except Exception as e:\n logger.error(\n \"Failed to destroy sandbox %s after registration failure: %s\",\n sandbox_id,\n e,\n )\n raise\n\n logger.info(f\"Created sandbox {sandbox_id} for thread {thread_id} at {info.sandbox_url}\")\n return sandbox_id\n\n def _check_tracked_sandbox_alive(self, sandbox_id: str, info: SandboxInfo) -> bool | None:\n \"\"\"Return whether a tracked sandbox appears alive, or None if unknown.\"\"\"\n try:\n return self._backend.is_alive(info)\n except Exception as e:\n logger.warning(f\"Failed to check sandbox {sandbox_id} health: {e}\")\n return None\n\n def _remove_tracked_sandbox(\n self,\n sandbox_id: str,\n *,\n expected_info: SandboxInfo | None = None,\n ) -> tuple[Sandbox | None, SandboxInfo | None, bool]:\n \"\"\"Remove a sandbox from in-process tracking maps.\n\n When expected_info is provided, removal only happens if the currently\n tracked active or warm-pool entry is the exact info object that was\n checked. This prevents a stale health-check result from deleting a\n freshly recreated sandbox with the same deterministic id.\n \"\"\"\n thread_keys_to_remove: list[tuple[str, str]] = []\n\n with self._lock:\n active_info = self._sandbox_infos.get(sandbox_id)\n warm_item = self._warm_pool.get(sandbox_id)\n warm_info = warm_item[0] if warm_item is not None else None\n if expected_info is not None and active_info is not expected_info and warm_info is not expected_info:\n return None, None, False\n\n sandbox = self._sandboxes.pop(sandbox_id, None)\n info = self._sandbox_infos.pop(sandbox_id, None)\n self._active_sandbox_identity.pop(sandbox_id, None)\n thread_keys_to_remove = [key for key, sid in self._thread_sandboxes.items() if sid == sandbox_id]\n for key in thread_keys_to_remove:\n del self._thread_sandboxes[key]\n self._last_activity.pop(sandbox_id, None)\n self._acquire_epoch.pop(sandbox_id, None)\n if info is None and sandbox_id in self._warm_pool:\n info, _ = self._warm_pool.pop(sandbox_id)\n else:\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n\n return sandbox, info, True\n\n def _drop_unhealthy_sandbox(self, sandbox_id: str, reason: str, *, expected_info: SandboxInfo | None = None) -> None:\n \"\"\"Remove and destroy a sandbox after a definitive failed health check.\"\"\"\n # Reserved for the whole path, not just the stop: this one untracks\n # first, so between the untrack and the `del:` claim an acquire misses\n # the caches and falls through to discovery, where `take()` still\n # succeeds against our own lease.\n if not self._reserve_local_teardown(sandbox_id, lambda: True):\n logger.info(f\"Skipped dropping sandbox {sandbox_id}: already being torn down by this instance\")\n return\n try:\n self._drop_unhealthy_reserved(sandbox_id, reason, expected_info=expected_info)\n finally:\n self._finish_local_teardown(sandbox_id)\n\n def _drop_unhealthy_reserved(self, sandbox_id: str, reason: str, *, expected_info: SandboxInfo | None = None) -> None:\n sandbox, info, removed = self._remove_tracked_sandbox(sandbox_id, expected_info=expected_info)\n if not removed:\n logger.info(f\"Skipped dropping sandbox {sandbox_id}: tracked info changed after health check\")\n return\n\n if sandbox is not None:\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing unhealthy sandbox {sandbox_id}: {e}\")\n\n if info is not None:\n # Gate this like every other reap path. The container failed a\n # definitive health check, but \"definitively dead to us\" is not proof\n # it is ours: a peer may have replaced the container behind this id,\n # in which case stopping it is the cross-instance kill again.\n if self._claim_ownership(sandbox_id, for_destroy=True):\n try:\n # Held like the other two stop paths: this one untracks before\n # claiming, so `_renew_owned_leases` cannot see the id either\n # and nothing else would refresh the marker. The heartbeat\n # releases the marker on exit (success or failure), so there is\n # no caller-side release to race a late refresh.\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(info)\n except Exception as e:\n logger.warning(f\"Error destroying unhealthy sandbox {sandbox_id}: {e}\")\n else:\n logger.info(\"Not destroying unhealthy sandbox %s: owned by another instance\", sandbox_id)\n\n logger.warning(f\"Dropped unhealthy sandbox {sandbox_id}: {reason}\")\n\n def _active_count_locked(self) -> int:\n \"\"\"Return active AIO sandbox count while ``_lock`` is held.\"\"\"\n return len(self._sandboxes)\n\n def _destroy_warm_entry(self, sandbox_id: str, entry: SandboxInfo, *, reason: str, still_reapable: Callable[[], bool]) -> bool:\n \"\"\"Destroy a warm-pool sandbox using AIO-specific backend logging.\n\n Claiming for destroy is the exclusion against **peers**: the lease is\n marked as a teardown, so a concurrent acquire on another instance is\n refused and the container cannot be re-acquired between this decision and\n the stop. That pairing is what replaced the per-sandbox flock guard. A\n claim that fails — peer-owned or backend unavailable — fails closed and\n we do not destroy.\n\n It is *not* an exclusion against this process: `claim()` succeeds against\n our own `own:` lease, so a same-process reclaim that ran before it wins\n the container and this stop lands on a turn already using it. The\n reservation is that half, and it is taken before the claim — after it,\n the entry stays visible in `_warm_pool` for the whole stop, so a reclaim\n would otherwise still find it.\n\n ``still_reapable`` is required rather than defaulting to unconditional:\n the safe default is the one that makes a new call site think about it,\n and this signature deliberately diverges from ``WarmPoolLifecycleMixin``'s\n hook for that reason. Safe because this provider overrides both mixin\n callers (``_evict_oldest_warm`` / ``_reap_expired_warm``); if those\n overrides were ever dropped, the mixin's call would fail loudly here\n rather than silently reopen the window.\n\n Returns:\n ``True`` when the container was stopped and the caller should drop\n its warm-pool entry; ``False`` when it is still running.\n \"\"\"\n if not self._reserve_local_teardown(sandbox_id, still_reapable):\n logger.info(\"Refusing to destroy warm-pool sandbox %s for %s: reclaimed by this instance\", sandbox_id, reason)\n return False\n\n try:\n if not self._claim_ownership(sandbox_id, for_destroy=True):\n logger.info(\"Refusing to destroy warm-pool sandbox %s for %s: owned by another instance\", sandbox_id, reason)\n return False\n\n try:\n # The marker must outlast the stop, not the TTL it was written with,\n # and is released by the heartbeat on exit. On a failed stop that\n # release matters just as much — the container is probably still up,\n # so a marker left behind would block its thread from re-acquiring it.\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(entry)\n except Exception as e:\n if reason == \"idle_timeout\":\n logger.error(f\"Failed to destroy idle warm-pool sandbox {sandbox_id}: {e}\")\n elif reason == \"replica_enforcement\":\n logger.error(f\"Failed to destroy warm-pool sandbox {sandbox_id}: {e}\")\n else:\n logger.error(f\"Failed to destroy warm-pool sandbox {sandbox_id} for {reason}: {e}\")\n return False\n\n # Remove the entry here, inside the reservation, rather than leaving\n # it to the caller. Releasing the reservation when the stop returns\n # and popping afterwards leaves a gap in which the container is\n # already stopped, the entry is still in `_warm_pool`, and nothing\n # marks it — so a reclaim picks it up and hands out a dead container.\n # The pop stays deferred relative to the *stop* (a refused or failed\n # stop keeps the entry), just no longer relative to the reservation.\n with self._lock:\n current = self._warm_pool.get(sandbox_id)\n if current is not None and current[0] is entry:\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n finally:\n self._finish_local_teardown(sandbox_id)\n\n if reason == \"idle_timeout\":\n logger.info(f\"Destroyed idle warm-pool sandbox {sandbox_id}\")\n elif reason == \"replica_enforcement\":\n logger.info(f\"Destroyed warm-pool sandbox {sandbox_id}\")\n else:\n logger.info(f\"Destroyed warm-pool sandbox {sandbox_id} for {reason}\")\n return True\n\n # ── Core: acquire / get / release / shutdown ─────────────────────────\n\n def acquire(self, thread_id: str | None = None, *, user_id: str | None = None) -> str:\n \"\"\"Acquire a sandbox environment and return its ID.\n\n For the same thread_id, this method will return the same sandbox_id\n across multiple turns, multiple processes, and (with shared storage)\n multiple pods.\n\n Thread-safe with both in-process and cross-process locking.\n\n Args:\n thread_id: Optional thread ID for thread-specific configurations.\n\n Returns:\n The ID of the acquired sandbox environment.\n \"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n if thread_id:\n thread_lock = self._get_thread_lock(thread_id, effective_user_id)\n with thread_lock:\n return self._acquire_internal(thread_id, user_id=effective_user_id)\n else:\n return self._acquire_internal(thread_id, user_id=effective_user_id)\n\n async def acquire_async(self, thread_id: str | None = None, *, user_id: str | None = None) -> str:\n \"\"\"Acquire a sandbox environment without blocking the event loop.\n\n Mirrors ``acquire()`` while keeping blocking backend operations off the\n event loop and using async-native readiness polling for newly created\n sandboxes.\n \"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n if thread_id:\n thread_lock = self._get_thread_lock(thread_id, effective_user_id)\n await _acquire_thread_lock_async(thread_lock)\n try:\n return await self._acquire_internal_async(thread_id, user_id=effective_user_id)\n finally:\n thread_lock.release()\n\n return await self._acquire_internal_async(thread_id, user_id=effective_user_id)\n\n def _acquire_internal(self, thread_id: str | None, *, user_id: str) -> str:\n \"\"\"Internal sandbox acquisition with two-layer consistency.\n\n Layer 1: In-process cache (fastest, covers same-process repeated access)\n Layer 2: Backend discovery (covers containers started by other processes;\n sandbox_id is deterministic from thread_id so no shared state file\n is needed — any process can derive the same container name)\n \"\"\"\n self._ensure_skills_projection(user_id)\n cached_id = self._reuse_in_process_sandbox(thread_id, user_id=user_id)\n if cached_id is not None:\n return cached_id\n\n # Deterministic ID for thread-specific, random for anonymous\n sandbox_id = self._sandbox_id_for_thread(thread_id, user_id)\n if thread_id:\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n self._assert_active_identity_available_locked(sandbox_id, key)\n\n # ── Layer 1.5: Warm pool (container still running, no cold-start) ──\n reclaimed_id = self._reclaim_warm_pool_sandbox(thread_id, sandbox_id, user_id=user_id)\n if reclaimed_id is not None:\n return reclaimed_id\n\n # ── Layer 2: Backend discovery + create (protected by cross-process lock) ──\n # Use a file lock so that two processes racing to create the same sandbox\n # for the same thread_id serialize here: the second process will discover\n # the container started by the first instead of hitting a name-conflict.\n if thread_id:\n return self._discover_or_create_with_lock(thread_id, sandbox_id, user_id=user_id)\n\n return self._create_sandbox(thread_id, sandbox_id, user_id=user_id)\n\n async def _acquire_internal_async(self, thread_id: str | None, *, user_id: str) -> str:\n \"\"\"Async counterpart to ``_acquire_internal``.\"\"\"\n await asyncio.to_thread(self._ensure_skills_projection, user_id)\n cached_id = await asyncio.to_thread(self._reuse_in_process_sandbox, thread_id, user_id=user_id)\n if cached_id is not None:\n return cached_id\n\n # Deterministic ID for thread-specific, random for anonymous\n sandbox_id = self._sandbox_id_for_thread(thread_id, user_id)\n if thread_id:\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n self._assert_active_identity_available_locked(sandbox_id, key)\n\n # ── Layer 1.5: Warm pool (container still running, no cold-start) ──\n reclaimed_id = await asyncio.to_thread(self._reclaim_warm_pool_sandbox, thread_id, sandbox_id, user_id=user_id)\n if reclaimed_id is not None:\n return reclaimed_id\n\n # ── Layer 2: Backend discovery + create (protected by cross-process lock) ──\n if thread_id:\n return await self._discover_or_create_with_lock_async(thread_id, sandbox_id, user_id=user_id)\n\n return await self._create_sandbox_async(thread_id, sandbox_id, user_id=user_id)\n\n def _discover_or_create_with_lock(self, thread_id: str, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Discover an existing sandbox or create a new one under a cross-process file lock.\n\n The file lock serializes concurrent sandbox creation for the same thread_id\n across multiple processes, preventing container-name conflicts.\n \"\"\"\n paths = get_paths()\n effective_user_id = self._effective_acquire_user_id(user_id)\n paths.ensure_thread_dirs(thread_id, user_id=effective_user_id)\n lock_path = paths.thread_dir(thread_id, user_id=effective_user_id) / f\"{sandbox_id}.lock\"\n\n with open(lock_path, \"a\", encoding=\"utf-8\") as lock_file:\n locked = False\n try:\n _lock_file_exclusive(lock_file)\n locked = True\n # Re-check in-process caches under the file lock in case another\n # thread in this process won the race while we were waiting.\n cached_id = self._recheck_cached_sandbox(thread_id, sandbox_id, user_id=effective_user_id)\n if cached_id is not None:\n return cached_id\n\n # Backend discovery: another process may have created the container.\n discovered = self._backend.discover(sandbox_id)\n if discovered is not None:\n return self._register_discovered_sandbox(thread_id, discovered, user_id=effective_user_id)\n\n return self._create_sandbox(thread_id, sandbox_id, user_id=effective_user_id)\n finally:\n if locked:\n _unlock_file(lock_file)\n\n async def _discover_or_create_with_lock_async(self, thread_id: str, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Async counterpart to ``_discover_or_create_with_lock``.\"\"\"\n paths = get_paths()\n effective_user_id = self._effective_acquire_user_id(user_id)\n await asyncio.to_thread(paths.ensure_thread_dirs, thread_id, user_id=effective_user_id)\n lock_path = paths.thread_dir(thread_id, user_id=effective_user_id) / f\"{sandbox_id}.lock\"\n\n lock_file = await asyncio.to_thread(_open_lock_file, lock_path)\n locked = False\n try:\n await asyncio.to_thread(_lock_file_exclusive, lock_file)\n locked = True\n # Re-check in-process caches under the file lock in case another\n # thread in this process won the race while we were waiting.\n cached_id = await asyncio.to_thread(self._recheck_cached_sandbox, thread_id, sandbox_id, user_id=effective_user_id)\n if cached_id is not None:\n return cached_id\n\n # Backend discovery is sync because local discovery may inspect\n # Docker and perform a health check; keep it off the event loop.\n discovered = await asyncio.to_thread(self._backend.discover, sandbox_id)\n if discovered is not None:\n # Registration publishes ownership, which is blocking store IO\n # (filesystem or network depending on the backend) — same reason\n # every other step in this coroutine is offloaded.\n return await asyncio.to_thread(self._register_discovered_sandbox, thread_id, discovered, user_id=effective_user_id)\n\n return await self._create_sandbox_async(thread_id, sandbox_id, user_id=effective_user_id)\n finally:\n if locked:\n await asyncio.to_thread(_unlock_file, lock_file)\n await asyncio.to_thread(lock_file.close)\n\n def _destroy_unready_sandbox(self, sandbox_id: str, info: SandboxInfo) -> None:\n \"\"\"Tear down a freshly-created container whose readiness check failed.\n\n The container was started by the backend but never reached ready, so it\n never entered ``_register_created_sandbox`` and the ownership store has\n no lease for it yet. For the full readiness timeout (60s) it runs\n unowned, which is exactly the window a peer gateway's startup\n reconciliation is built to adopt across (#4206). Without a claim, a peer\n that adopts the not-yet-ready Pod and then has this instance's stop land\n on it is a cross-instance kill that interrupts an active turn (#4248).\n\n Claim the teardown lease first so this reap path is gated by the same\n ownership guard as every other destroy (``_destroy_warm_entry``,\n ``_drop_unhealthy_reserved``); fail closed (leave the container for the\n peer to reap via its own reconciliation) if a peer already owns it or\n the ownership store cannot answer.\n\n The claim alone is only the cross-**instance** half: it succeeds against\n our own lease by design, so it says nothing about this process. The\n same-process half is the local teardown reservation, taken first and\n held across the whole path — between the readiness timeout and the\n claim, ``_reconcile_orphans`` (idle checker, every 60s) can see this\n container running, untracked, and past its recovery grace, and park it\n in ``_warm_pool``; the subsequent claim would still succeed and the\n stop would land on an entry this instance has just adopted, leaving a\n dead warm entry for the next reclaim to hand out. The predicate checks\n the id is absent from both the active and warm maps; the reservation\n makes that check and the teardown mark one critical section, so no\n adopt/acquire can slip between them (same pairing as\n ``_destroy_warm_entry``).\n \"\"\"\n if not self._reserve_local_teardown(\n sandbox_id,\n lambda: sandbox_id not in self._sandboxes and sandbox_id not in self._sandbox_infos and sandbox_id not in self._warm_pool,\n ):\n logger.warning(\n \"Not destroying unready sandbox %s: adopted or being torn down by this instance\",\n sandbox_id,\n )\n return\n try:\n if not self._claim_ownership(sandbox_id, for_destroy=True):\n logger.warning(\n \"Not destroying unready sandbox %s: owned by another instance or ownership unavailable\",\n sandbox_id,\n )\n return\n try:\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(info)\n except Exception as e:\n logger.warning(f\"Error destroying unready sandbox {sandbox_id}: {e}\")\n finally:\n self._finish_local_teardown(sandbox_id)\n\n def _create_sandbox(self, thread_id: str | None, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Create a new sandbox via the backend.\n\n Args:\n thread_id: Optional thread ID.\n sandbox_id: The sandbox ID to use.\n\n Returns:\n The sandbox_id.\n\n Raises:\n RuntimeError: If sandbox creation or readiness check fails.\n \"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n extra_mounts = self._get_extra_mounts(thread_id, user_id=effective_user_id)\n provision_lark_cli_runtime = self._lark_integration_active(effective_user_id)\n provision_lark_cli_broker = self._lark_broker_active(effective_user_id)\n\n # Enforce replicas: only warm-pool containers count toward eviction budget.\n # Active sandboxes are in use by live threads and must not be forcibly stopped.\n replicas, total = self._replica_count()\n if total >= replicas:\n evicted = self._evict_oldest_warm()\n self._log_replicas_soft_cap(replicas, sandbox_id, evicted)\n\n info = self._backend.create(\n thread_id,\n sandbox_id,\n extra_mounts=extra_mounts or None,\n user_id=effective_user_id,\n provision_lark_cli_runtime=provision_lark_cli_runtime,\n provision_lark_cli_broker=provision_lark_cli_broker,\n )\n\n # Wait for sandbox to be ready\n if not wait_for_sandbox_ready(info.sandbox_url, timeout=60):\n # The container is running but unowned: ownership is published by\n # ``_register_created_sandbox`` after this gate. Claim the teardown\n # lease before stopping it so a peer cannot adopt the not-yet-ready\n # Pod in the meantime (#4248).\n self._destroy_unready_sandbox(sandbox_id, info)\n raise RuntimeError(f\"Sandbox {sandbox_id} failed to become ready within timeout at {info.sandbox_url}\")\n\n return self._register_created_sandbox(thread_id, sandbox_id, info, user_id=effective_user_id)\n\n async def _create_sandbox_async(self, thread_id: str | None, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Async counterpart to ``_create_sandbox``.\"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n extra_mounts = await asyncio.to_thread(self._get_extra_mounts, thread_id, user_id=effective_user_id)\n provision_lark_cli_runtime = await asyncio.to_thread(self._lark_integration_active, effective_user_id)\n provision_lark_cli_broker = await asyncio.to_thread(self._lark_broker_active, effective_user_id)\n\n # Enforce replicas: only warm-pool containers count toward eviction budget.\n # Active sandboxes are in use by live threads and must not be forcibly stopped.\n replicas, total = self._replica_count()\n if total >= replicas:\n evicted = await asyncio.to_thread(self._evict_oldest_warm)\n self._log_replicas_soft_cap(replicas, sandbox_id, evicted)\n\n info = await asyncio.to_thread(\n self._backend.create,\n thread_id,\n sandbox_id,\n extra_mounts=extra_mounts or None,\n user_id=effective_user_id,\n provision_lark_cli_runtime=provision_lark_cli_runtime,\n provision_lark_cli_broker=provision_lark_cli_broker,\n )\n\n # Wait for sandbox to be ready without blocking the event loop.\n if not await wait_for_sandbox_ready_async(info.sandbox_url, timeout=60):\n # The container is running but unowned: ownership is published by\n # ``_register_created_sandbox`` after this gate. Claim the teardown\n # lease before stopping it so a peer cannot adopt the not-yet-ready\n # Pod in the meantime (#4248).\n await asyncio.to_thread(self._destroy_unready_sandbox, sandbox_id, info)\n raise RuntimeError(f\"Sandbox {sandbox_id} failed to become ready within timeout at {info.sandbox_url}\")\n\n # Registration publishes ownership (blocking store IO), so it is offloaded\n # like every other blocking step on this path.\n return await asyncio.to_thread(self._register_created_sandbox, thread_id, sandbox_id, info, user_id=effective_user_id)\n\n def get(self, sandbox_id: str) -> Sandbox | None:\n \"\"\"Get a sandbox by ID. Updates last activity timestamp.\n\n Stays a pure in-memory lookup: async tool paths call this directly on the\n event loop (``ensure_sandbox_initialized_async``), so it must not touch\n the ownership store — that is blocking filesystem or network IO depending\n on the backend. Ownership is published off the event loop on\n acquire/reclaim and refreshed by the renewal thread (see\n ``_renew_owned_leases``).\n\n Args:\n sandbox_id: The ID of the sandbox.\n\n Returns:\n The sandbox instance if found, None otherwise.\n \"\"\"\n with self._lock:\n sandbox = self._sandboxes.get(sandbox_id)\n if sandbox is not None:\n self._last_activity[sandbox_id] = time.time()\n return sandbox\n\n def release(self, sandbox_id: str) -> None:\n \"\"\"Release a sandbox from active use into the warm pool.\n\n The container is kept running so it can be reclaimed quickly by the same\n thread on its next turn without a cold-start. The container will only be\n stopped when the replicas limit forces eviction or during shutdown.\n\n The host-side HTTP client owned by the cached ``AioSandbox`` instance is\n closed before the instance is dropped (#2872). The warm-pool entry only\n stores ``SandboxInfo``, so a fresh ``AioSandbox`` (and a fresh client)\n is constructed if the container is later reclaimed.\n\n Args:\n sandbox_id: The ID of the sandbox to release.\n \"\"\"\n info = None\n sandbox = None\n thread_keys_to_remove: list[tuple[str, str]] = []\n\n with self._lock:\n sandbox = self._sandboxes.pop(sandbox_id, None)\n info = self._sandbox_infos.pop(sandbox_id, None)\n thread_keys_to_remove = [key for key, sid in self._thread_sandboxes.items() if sid == sandbox_id]\n for key in thread_keys_to_remove:\n del self._thread_sandboxes[key]\n active_identity = self._active_sandbox_identity.pop(sandbox_id, None)\n self._last_activity.pop(sandbox_id, None)\n # Park in warm pool — container keeps running\n if info and sandbox_id not in self._warm_pool:\n self._warm_pool[sandbox_id] = (info, time.time())\n self._warm_pool_identity[sandbox_id] = thread_keys_to_remove[0] if thread_keys_to_remove else active_identity\n\n if sandbox is not None:\n # Defense-in-depth: close() already swallows its own errors; this\n # guard only protects against a future close() that misbehaves, so\n # host-side client cleanup can never block parking in the warm pool.\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} during release: {e}\")\n\n # Keep the lease while warm so a peer cannot adopt+destroy before we\n # reclaim, re-establishing it if it lapsed during a long turn. Never\n # raises: the turn is already over, so a store problem must not surface\n # through after_agent, and the renewal thread (which covers warm entries)\n # is the actual guarantee — this only narrows the window.\n if info is not None:\n # Same staleness as the renewal thread: the refresh is a store round\n # trip, and the thread's next turn can reclaim this warm entry while\n # it is in flight. Only drop it if nothing re-acquired it since.\n epoch = self._acquire_epoch_of(sandbox_id)\n if not self._refresh_ownership(sandbox_id):\n logger.warning(\"Sandbox %s is owned by another instance; releasing it from this warm pool\", sandbox_id)\n self._forget_lost_sandbox(sandbox_id, expected_epoch=epoch)\n\n logger.info(f\"Released sandbox {sandbox_id} to warm pool (container still running)\")\n\n def destroy(self, sandbox_id: str) -> None:\n \"\"\"Destroy a sandbox: stop the container and free all resources.\n\n Unlike release(), this actually stops the container. Use this for\n explicit cleanup, capacity-driven eviction, or shutdown.\n\n The host-side HTTP client owned by the cached ``AioSandbox`` instance is\n closed alongside backend/container destruction so no client/socket\n resources leak (#2872).\n\n Args:\n sandbox_id: The ID of the sandbox to destroy.\n \"\"\"\n self._destroy_tracked(sandbox_id, still_reapable=lambda: True)\n\n def _destroy_tracked(self, sandbox_id: str, *, still_reapable: Callable[[], bool]) -> None:\n \"\"\"``destroy()`` with a caller-supplied \"is this still reapable\" gate.\n\n Callers that decided to destroy *earlier* (the idle checker) pass their\n own predicate so the decision is re-validated in the same critical\n section that reserves the teardown. ``destroy()`` itself passes a\n constant: an explicit destroy is a decision made now.\n \"\"\"\n if not self._reserve_local_teardown(sandbox_id, still_reapable):\n logger.info(\"Skipping destroy of sandbox %s: re-acquired by this instance or already being torn down\", sandbox_id)\n return\n\n try:\n self._destroy_reserved(sandbox_id)\n finally:\n self._finish_local_teardown(sandbox_id)\n\n def _destroy_reserved(self, sandbox_id: str) -> None:\n # Claim before untracking. The reverse order loses the container on a\n # refused claim: still running, and no longer in any of our maps, so\n # nothing here would ever reap or reclaim it.\n if not self._claim_ownership(sandbox_id, for_destroy=True):\n logger.warning(\"Refusing to destroy sandbox %s: owned by another instance\", sandbox_id)\n return\n\n sandbox, info, _ = self._remove_tracked_sandbox(sandbox_id)\n\n if sandbox is not None:\n # Defense-in-depth: close() already swallows its own errors; this\n # guard only protects against a future close() that misbehaves, so\n # host-side client cleanup can never block container destruction.\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} during destroy: {e}\")\n\n if info:\n # The marker must outlast the stop, not the TTL it was written with,\n # and the heartbeat releases it on exit — on both outcomes. On a\n # failed stop the container is probably still up, so a marker left\n # behind would refuse its own thread's `take()` until the TTL lapses;\n # the error still propagates out of the `with` (`shutdown()` logs per\n # sandbox off it), it is just no longer this method's job to release.\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(info)\n logger.info(f\"Destroyed sandbox {sandbox_id}\")\n else:\n # No container to stop, so no teardown lease was held: clear the\n # marker the claim above wrote, so an untracked id cannot leave a\n # lease stuck in `del:`.\n self._release_ownership(sandbox_id)\n\n def shutdown(self) -> None:\n \"\"\"Shutdown all sandboxes. Thread-safe and idempotent.\"\"\"\n with self._lock:\n if self._shutdown_called:\n return\n self._shutdown_called = True\n sandbox_ids = list(self._sandboxes.keys())\n warm_items = list(self._warm_pool.items())\n self._warm_pool.clear()\n self._warm_pool_identity.clear()\n\n self._stop_idle_checker()\n # Stop renewing before destroying: the destroy paths claim ownership\n # themselves, and a renewal racing them only re-publishes leases we are\n # about to drop.\n self._stop_lease_renewal()\n\n logger.info(f\"Shutting down {len(sandbox_ids)} active + {len(warm_items)} warm-pool sandbox(es)\")\n\n for sandbox_id in sandbox_ids:\n try:\n self.destroy(sandbox_id)\n except Exception as e:\n logger.error(f\"Failed to destroy sandbox {sandbox_id} during shutdown: {e}\")\n\n for sandbox_id, (info, _) in warm_items:\n # Route through _destroy_warm_entry so the ownership claim and the\n # container stop stay together, as on the idle path. Unconditional\n # here: the entries were removed from `_warm_pool` under the lock\n # above, so the pool-membership predicate the other callers use would\n # refuse every one of them.\n self._destroy_warm_entry(sandbox_id, info, reason=\"shutdown\", still_reapable=lambda: True)\n\n try:\n self._ownership.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox ownership store during shutdown: {e}\")\n"}, "bec6277930d6ee73c58156689f1556780724e35d": {"t": 1785846864.4262247, "n": 115342, "h": "72a9842ecddaaaf9", "c": "\"\"\"AIO Sandbox Provider — orchestrates sandbox lifecycle with pluggable backends.\n\nThis provider composes:\n- SandboxBackend: how sandboxes are provisioned (local container vs remote/K8s)\n\nThe provider itself handles:\n- In-process caching for fast repeated access\n- Idle timeout management\n- Graceful shutdown with signal handling\n- Mount computation (thread-specific, skills)\n\"\"\"\n\nimport asyncio\nimport atexit\nimport contextlib\nimport hashlib\nimport logging\nimport os\nimport signal\nimport threading\nimport time\nimport uuid\nfrom collections.abc import Callable\nfrom concurrent.futures import ThreadPoolExecutor\n\ntry:\n import fcntl\nexcept ImportError: # pragma: no cover - Windows fallback\n fcntl = None # type: ignore[assignment]\n import msvcrt\n\nfrom deerflow.community.warm_pool_lifecycle import (\n DEFAULT_IDLE_TIMEOUT,\n DEFAULT_REPLICAS,\n WarmPoolLifecycleMixin,\n)\nfrom deerflow.community.warm_pool_lifecycle import (\n IDLE_CHECK_INTERVAL as _SHARED_IDLE_CHECK_INTERVAL,\n)\nfrom deerflow.config import get_app_config\nfrom deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths, join_host_path\nfrom deerflow.integrations.lark_cli import INTEGRATION_ID as LARK_CLI_INTEGRATION_ID\nfrom deerflow.integrations.lark_cli import LARK_CLI_SANDBOX_CONFIG_DIR, LARK_CLI_SANDBOX_DATA_DIR, LARK_CLI_SANDBOX_RUNTIME_DIR, ensure_lark_cli_credential_tree, lark_skills_installed\nfrom deerflow.runtime.user_context import get_effective_user_id\nfrom deerflow.sandbox.sandbox import Sandbox\nfrom deerflow.sandbox.sandbox_provider import SandboxProvider\n\nfrom .aio_sandbox import AioSandbox\nfrom .backend import SandboxBackend, wait_for_sandbox_ready, wait_for_sandbox_ready_async\nfrom .local_backend import LocalContainerBackend\nfrom .ownership import (\n OwnershipBackendError,\n RenewOutcome,\n SandboxOwnershipStore,\n compute_lease_ttl,\n generate_owner_id,\n make_sandbox_ownership_store,\n resolve_ownership_config,\n)\nfrom .remote_backend import RemoteSandboxBackend\nfrom .sandbox_info import SandboxInfo\n\nlogger = logging.getLogger(__name__)\n\n# Default configuration\nDEFAULT_IMAGE = \"enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest\"\nDEFAULT_PORT = 8080\nDEFAULT_CONTAINER_PREFIX = \"deer-flow-sandbox\"\nIDLE_CHECK_INTERVAL = _SHARED_IDLE_CHECK_INTERVAL\nTHREAD_LOCK_EXECUTOR_WORKERS = min(32, (os.cpu_count() or 1) + 4)\n_THREAD_LOCK_EXECUTOR = ThreadPoolExecutor(max_workers=THREAD_LOCK_EXECUTOR_WORKERS, thread_name_prefix=\"sandbox-lock-wait\")\natexit.register(_THREAD_LOCK_EXECUTOR.shutdown, wait=False, cancel_futures=True)\n\n\nclass SandboxBeingDestroyedError(RuntimeError):\n \"\"\"A peer is tearing this container down, so it must not be handed out.\n\n Raised on the acquire path when the ownership lease is in its teardown state.\n The caller drops the container from tracking and lets the normal\n discover-or-create path provision a fresh one, rather than handing an agent a\n sandbox that is about to stop underneath it.\n \"\"\"\n\n def __init__(self, sandbox_id: str) -> None:\n super().__init__(f\"sandbox {sandbox_id} is being destroyed by another instance\")\n self.sandbox_id = sandbox_id\n\n\nclass SandboxIdentityCollisionError(RuntimeError):\n \"\"\"A deterministic ID is already tracked for a different user/thread.\"\"\"\n\n def __init__(\n self,\n sandbox_id: str,\n stored_key: tuple[str, str] | None,\n requested_key: tuple[str, str],\n ) -> None:\n super().__init__(f\"sandbox ID collision for {sandbox_id}: tracked identity is {stored_key!r}, requested identity is {requested_key!r}\")\n self.sandbox_id = sandbox_id\n\n\ndef _lock_file_exclusive(lock_file) -> None:\n if fcntl is not None:\n fcntl.flock(lock_file, fcntl.LOCK_EX)\n return\n\n lock_file.seek(0)\n msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)\n\n\ndef _unlock_file(lock_file) -> None:\n if fcntl is not None:\n fcntl.flock(lock_file, fcntl.LOCK_UN)\n return\n\n lock_file.seek(0)\n msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)\n\n\ndef _open_lock_file(lock_path):\n return open(lock_path, \"a\", encoding=\"utf-8\")\n\n\nasync def _acquire_thread_lock_async(lock: threading.Lock) -> None:\n \"\"\"Acquire a threading.Lock without polling or using the default executor.\"\"\"\n loop = asyncio.get_running_loop()\n acquire_future = loop.run_in_executor(_THREAD_LOCK_EXECUTOR, lock.acquire, True)\n\n try:\n acquired = await asyncio.shield(acquire_future)\n except asyncio.CancelledError:\n acquire_future.add_done_callback(lambda task: _release_cancelled_lock_acquire(lock, task))\n raise\n\n if not acquired:\n raise RuntimeError(\"Failed to acquire sandbox thread lock\")\n\n\ndef _release_cancelled_lock_acquire(lock: threading.Lock, task: asyncio.Future[bool]) -> None:\n \"\"\"Release a lock acquired after its awaiting coroutine was cancelled.\"\"\"\n if task.cancelled():\n return\n\n try:\n acquired = task.result()\n except Exception as e:\n logger.warning(f\"Cancelled sandbox lock acquisition finished with error: {e}\")\n return\n\n if acquired:\n lock.release()\n\n\nclass AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):\n \"\"\"Sandbox provider that manages containers running the AIO sandbox.\n\n Architecture:\n This provider composes a SandboxBackend (how to provision), enabling:\n - Local Docker/Apple Container mode (auto-start containers)\n - Remote/K8s mode (connect to pre-existing sandbox URL)\n\n Configuration options in config.yaml under sandbox:\n use: deerflow.community.aio_sandbox:AioSandboxProvider\n image: \n port: 8080 # Base port for local containers\n container_prefix: deer-flow-sandbox\n idle_timeout: 600 # Idle timeout in seconds (0 to disable)\n replicas: 3 # Max concurrent sandbox containers (LRU eviction when exceeded)\n thread_data_mounts: null # null = backend auto-detection\n mounts: # Volume mounts for local containers\n - host_path: /path/on/host\n container_path: /path/in/container\n read_only: false\n environment: # Environment variables for containers\n NODE_ENV: production\n API_KEY: $MY_API_KEY\n \"\"\"\n\n # How long `_held_teardown_lease` waits for its heartbeat thread to exit\n # before deferring the final lease release to that (still-running) thread.\n # The store's socket timeout bounds each operation, but context exit can\n # catch the heartbeat in one final refresh and must then wait for its final\n # release. Keep this above both sequential five-second operation bounds so a\n # normally timing-out refresh + release still finishes synchronously.\n _TEARDOWN_JOIN_TIMEOUT_SECONDS = 12.0\n\n def __init__(self):\n self._lock = threading.Lock()\n self._sandboxes: dict[str, AioSandbox] = {} # sandbox_id -> AioSandbox instance\n self._sandbox_infos: dict[str, SandboxInfo] = {} # sandbox_id -> SandboxInfo (for destroy)\n self._thread_sandboxes: dict[tuple[str, str], str] = {} # (user_id, thread_id) -> sandbox_id\n self._thread_locks: dict[tuple[str, str], threading.Lock] = {} # (user_id, thread_id) -> in-process lock\n self._last_activity: dict[str, float] = {} # sandbox_id -> last activity timestamp\n # Warm pool: released sandboxes whose containers are still running.\n # Maps sandbox_id -> (SandboxInfo, release_timestamp).\n # Containers here can be reclaimed quickly (no cold-start) or destroyed\n # when replicas capacity is exhausted.\n self._warm_pool: dict[str, tuple[SandboxInfo, float]] = {}\n self._active_sandbox_identity: dict[str, tuple[str, str] | None] = {}\n self._warm_pool_identity: dict[str, tuple[str, str] | None] = {}\n # sandbox_id -> when reconciliation first saw it running with no lease.\n # Gates adoption behind a recovery grace (see _adoptable_after_grace).\n self._unowned_since: dict[str, float] = {}\n # The two halves of same-process exclusion. The ownership store excludes\n # peers and nothing else — `claim()` and `take()` both succeed against\n # our own lease by design — so `del:` says nothing to this process's own\n # threads. See _reserve_local_teardown / _acquire_epoch.\n self._local_teardown: set[str] = set()\n self._acquire_epoch: dict[str, int] = {}\n self._acquire_epoch_counter = 0\n self._acquire_inflight: dict[str, int] = {}\n self._shutdown_called = False\n self._idle_checker_stop = threading.Event()\n self._idle_checker_thread: threading.Thread | None = None\n self._renewal_stop = threading.Event()\n self._renewal_thread: threading.Thread | None = None\n # Per-instance id used for cross-instance sandbox ownership leases (#4206).\n self._owner_id = generate_owner_id()\n\n self._config = self._load_config()\n self._ownership_config = resolve_ownership_config(self._config.get(\"ownership\"), stream_bridge=self._config.get(\"stream_bridge\"))\n self._ownership: SandboxOwnershipStore = make_sandbox_ownership_store(self._ownership_config, owner_id=self._owner_id)\n if not self._ownership.supports_cross_process:\n # Peers cannot see these leases, so every container looks like an\n # orphan to them. Say so once rather than letting #4206 resurface\n # silently on a multi-worker deployment that never set the config.\n logger.warning(\n \"Sandbox ownership store cannot coordinate across processes (sandbox.ownership.type: %s). \"\n \"Safe for a single gateway instance only — multi-worker / load-balanced gateways sharing a \"\n \"container backend must set sandbox.ownership.type: redis, or peers will adopt and idle-destroy \"\n \"each other's live sandboxes (#4206).\",\n self._ownership_config.type,\n )\n self._backend: SandboxBackend = self._create_backend()\n\n # Register shutdown handler\n atexit.register(self.shutdown)\n self._register_signal_handlers()\n\n # Reconcile orphaned containers from previous process lifecycles\n self._reconcile_orphans()\n\n # Renewal is independent of idle cleanup: an owner must keep proving it is\n # alive even when the idle reaper is disabled, or peers adopt its live\n # containers once the lease lapses (idle_timeout: 0 is a supported config).\n self._start_lease_renewal()\n\n # Start idle checker if enabled\n if self._config.get(\"idle_timeout\", DEFAULT_IDLE_TIMEOUT) > 0:\n self._start_idle_checker()\n\n @property\n def uses_thread_data_mounts(self) -> bool:\n \"\"\"Whether thread workspace/uploads/outputs are visible via mounts.\n\n Local container backends bind-mount the thread data directories, so files\n written by the gateway are already visible when the sandbox starts.\n Remote backends may require explicit file sync. Operators can override\n this detection when gateway and remote sandboxes share the same storage.\n \"\"\"\n override = self._config.get(\"thread_data_mounts\")\n if override is not None:\n return override\n return isinstance(self._backend, LocalContainerBackend)\n\n # ── Factory methods ──────────────────────────────────────────────────\n\n def _create_backend(self) -> SandboxBackend:\n \"\"\"Create the appropriate backend based on configuration.\n\n Selection logic (checked in order):\n 1. ``provisioner_url`` set → RemoteSandboxBackend (provisioner mode)\n Provisioner dynamically creates Pods + Services in k3s.\n 2. Default → LocalContainerBackend (local mode)\n Local provider manages container lifecycle directly (start/stop).\n \"\"\"\n provisioner_url = self._config.get(\"provisioner_url\")\n if provisioner_url:\n logger.info(f\"Using remote sandbox backend with provisioner at {provisioner_url}\")\n api_key = self._config.get(\"provisioner_api_key\", \"\")\n return RemoteSandboxBackend(provisioner_url=provisioner_url, api_key=api_key)\n\n logger.info(\"Using local container sandbox backend\")\n return LocalContainerBackend(\n image=self._config[\"image\"],\n base_port=self._config[\"port\"],\n container_prefix=self._config[\"container_prefix\"],\n config_mounts=self._config[\"mounts\"],\n environment=self._config[\"environment\"],\n )\n\n # ── Configuration ────────────────────────────────────────────────────\n\n def _load_config(self) -> dict:\n \"\"\"Load sandbox configuration from app config.\"\"\"\n config = get_app_config()\n sandbox_config = config.sandbox\n\n idle_timeout = getattr(sandbox_config, \"idle_timeout\", None)\n replicas = getattr(sandbox_config, \"replicas\", None)\n\n return {\n \"image\": sandbox_config.image or DEFAULT_IMAGE,\n \"port\": sandbox_config.port or DEFAULT_PORT,\n \"container_prefix\": sandbox_config.container_prefix or DEFAULT_CONTAINER_PREFIX,\n \"idle_timeout\": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,\n \"replicas\": replicas if replicas is not None else DEFAULT_REPLICAS,\n \"mounts\": sandbox_config.mounts or [],\n \"thread_data_mounts\": getattr(sandbox_config, \"thread_data_mounts\", None),\n \"environment\": self._resolve_env_vars(sandbox_config.environment or {}),\n \"ownership\": getattr(sandbox_config, \"ownership\", None),\n # A redis stream bridge means the deployment is multi-instance, which\n # is what the ownership store must default to. Read the same source\n # the bridge's own resolver reads, not just its env var.\n \"stream_bridge\": getattr(config, \"stream_bridge\", None),\n # provisioner URL for dynamic pod management (e.g. http://provisioner:8002)\n \"provisioner_url\": getattr(sandbox_config, \"provisioner_url\", None) or \"\",\n \"provisioner_api_key\": getattr(sandbox_config, \"provisioner_api_key\", None) or \"\",\n }\n\n @staticmethod\n def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:\n \"\"\"Resolve environment variable references (values starting with $).\"\"\"\n resolved = {}\n for key, value in env_config.items():\n if isinstance(value, str) and value.startswith(\"$\"):\n env_name = value[1:]\n resolved[key] = os.environ.get(env_name, \"\")\n else:\n resolved[key] = str(value)\n return resolved\n\n # ── Cross-instance ownership leases ───────────────────────────────────\n\n def _publish_ownership(self, sandbox_id: str) -> None:\n \"\"\"Take responsibility for *sandbox_id* on the acquire path.\n\n Takes over from whichever instance served this thread last — the\n container is deterministic per (user, thread), so a turn routing here is\n a legitimate handover. The previous owner's next renewal reports LOST and\n it stops tracking the container without touching it.\n\n Deliberately **not** fail-open. Swallowing the error and handing the\n sandbox out anyway would leave it unowned while in active use, so a peer\n would see an orphan and reap it — the exact failure this store exists to\n stop. Callers must let this propagate.\n\n The intent mark is published **before** the round trip, and that ordering\n is the point. ``take()`` makes the takeover durable before it returns —\n on redis the server has committed the SET while the reply is still in\n flight — so bumping the epoch afterwards leaves a window where the store\n already says the container is ours but the guard still reads as though it\n were not. A renewal holding an older ``LOST`` walks straight through it,\n drops the maps, and closes the client this call is about to hand back, so\n acquire returns an id the provider no longer tracks and ``get()`` answers\n ``None``. A guard must become visible no later than the transition it\n guards; the epoch alone cannot, because it can only be written after the\n call that performs the transition returns.\n\n So the two marks cover the two halves and are both needed: the intent\n mark covers \"an acquire is mid-flight\", the epoch covers \"an acquire\n completed since you decided\".\n\n Raises:\n SandboxBeingDestroyedError: a peer is tearing this container down, so\n it must not be handed to an agent (the destroy → re-acquire race).\n OwnershipBackendError: ownership could not be published.\n \"\"\"\n with self._lock:\n self._acquire_inflight[sandbox_id] = self._acquire_inflight.get(sandbox_id, 0) + 1\n try:\n if not self._ownership.take(sandbox_id):\n raise SandboxBeingDestroyedError(sandbox_id)\n with self._lock:\n self._acquire_epoch_counter += 1\n self._acquire_epoch[sandbox_id] = self._acquire_epoch_counter\n finally:\n # A count rather than a set: acquires for one id are serialized by\n # the per-thread lock today, so a set would be equivalent — but that\n # is an assumption about a caller two layers up, and if it ever\n # stopped holding, a set would be cleared by the first finisher and\n # silently reopen this window. Counting removes the assumption.\n with self._lock:\n remaining = self._acquire_inflight.get(sandbox_id, 0) - 1\n if remaining > 0:\n self._acquire_inflight[sandbox_id] = remaining\n else:\n self._acquire_inflight.pop(sandbox_id, None)\n\n # ── Same-process exclusion (the half the store does not provide) ───────\n #\n # A lease excludes *peers*: `claim()` succeeds against our own `own:` lease\n # by design (that is what lets a destroy path claim what it already owns),\n # and `take()` succeeds against it too. So between this process's reaper\n # threads — idle checker, renewal, eviction — and its own acquire path, the\n # store provides **no exclusion at all**. Every reaper decides outside\n # `self._lock` (a store round trip must not be held under the lock that\n # guards every acquire), so each one acts on a decision an acquire may\n # already have invalidated. The two helpers below are that missing half, one\n # per direction:\n #\n # reaping — we are about to stop/drop it, so nothing may promote it:\n # reserve it, and make every promote path honour the reservation\n # exactly as it honours a peer's `del:`.\n # forgetting — a peer legitimately owns it and must win, so the promote is\n # the thing to detect: compare the acquire epoch we decided on.\n\n def _reserve_local_teardown(self, sandbox_id: str, still_reapable: Callable[[], bool]) -> bool:\n \"\"\"Reserve *sandbox_id* for teardown by this process.\n\n ``still_reapable`` is evaluated in the **same** critical section as the\n reservation, so no acquire can slip between the last check and the mark.\n That pairing is the whole point: checking first and marking second is the\n window, not a narrower version of it.\n\n Consequence, and the one rule a new caller has to know: **the predicate\n runs with ``self._lock`` held**, which is a plain ``Lock``, so a predicate\n that touches the lock — directly, or via a provider method that takes it —\n deadlocks. Predicates must be cheap, non-blocking reads of the maps\n (``sandbox_id in self._warm_pool``, a ``_last_activity`` comparison). The\n constraint is stated rather than engineered around on purpose: making the\n lock reentrant to tolerate it would trade a loud hang for a quiet class of\n re-entrancy bugs everywhere else in this provider.\n \"\"\"\n with self._lock:\n if sandbox_id in self._local_teardown or not still_reapable():\n return False\n self._local_teardown.add(sandbox_id)\n return True\n\n def _finish_local_teardown(self, sandbox_id: str) -> None:\n with self._lock:\n self._local_teardown.discard(sandbox_id)\n\n def _being_torn_down_locally(self, sandbox_id: str) -> bool:\n \"\"\"Whether a reaper thread in *this* process holds *sandbox_id*.\n\n Callers must already hold ``self._lock``.\n \"\"\"\n return sandbox_id in self._local_teardown\n\n def _acquire_epoch_of(self, sandbox_id: str) -> int:\n \"\"\"Snapshot the acquire generation, so a stale decision can be detected.\n\n Bumped only by ``_publish_ownership`` — i.e. exactly when an acquire path\n (re)takes the lease on the way to handing the sandbox to an agent.\n Re-establishing a lapsed lease from ``_refresh_ownership`` deliberately\n does not bump it: nothing was handed out, so a reaper's decision about\n that id is still current.\n \"\"\"\n with self._lock:\n return self._acquire_epoch.get(sandbox_id, 0)\n\n def _claim_ownership(self, sandbox_id: str, *, for_destroy: bool = False) -> bool:\n \"\"\"Take (or refresh) ownership of *sandbox_id*.\n\n A successful claim is what makes acting on the container safe: while we\n hold the lease a peer's claim fails. With ``for_destroy`` the lease is\n additionally marked as a teardown, which a concurrent acquire-side\n ``take()`` refuses — that is what closes the ownership-check → container-\n stop window the deleted per-sandbox flock guard used to cover.\n\n Fails closed on a backend error: ownership unknown is treated as\n \"not ours\" so we neither adopt nor destroy the container.\n \"\"\"\n try:\n return self._ownership.claim(sandbox_id, for_destroy=for_destroy)\n except OwnershipBackendError as e:\n logger.warning(\"Sandbox ownership claim failed for %s (treating as not owned): %s\", sandbox_id, e)\n return False\n\n def _release_ownership(self, sandbox_id: str) -> None:\n try:\n self._ownership.release(sandbox_id)\n except OwnershipBackendError as e:\n # Best effort: the lease expires on its own, so a failed release\n # delays reuse rather than corrupting ownership.\n logger.warning(\"Failed to release sandbox ownership for %s: %s\", sandbox_id, e)\n\n def _refresh_ownership(self, sandbox_id: str) -> bool:\n \"\"\"Keep holding *sandbox_id*'s lease. False when a peer has taken it.\n\n A **lapsed** lease is re-established rather than treated as lost: nobody\n holds it, so re-claiming is safe, and this is what keeps a Redis restart\n (which drops every key) from evicting every live sandbox fleet-wide. A\n lease a peer actually holds is never re-taken — that is the #4206 kill.\n \"\"\"\n try:\n outcome = self._ownership.renew(sandbox_id)\n except OwnershipBackendError as e:\n # Unknown, not lost: keep the sandbox and retry next tick. The TTL\n # still bounds how long a genuinely dead owner holds the lease.\n logger.warning(\"Could not renew sandbox ownership for %s, will retry: %s\", sandbox_id, e)\n return True\n\n if outcome is RenewOutcome.RENEWED:\n return True\n if outcome is RenewOutcome.LAPSED:\n # Free: re-establish. This is the deliberate fail-open renewal path,\n # so it cannot use `_claim_ownership`: that helper turns a backend\n # error into False for adopt/reap callers, which would conflate an\n # outage between these two round trips with a peer takeover and\n # evict a live sandbox. Unknown means keep-and-retry here, exactly as\n # it does when the `renew()` call itself cannot answer above.\n try:\n if self._ownership.claim(sandbox_id):\n logger.info(\"Re-established a lapsed ownership lease for %s\", sandbox_id)\n return True\n except OwnershipBackendError as e:\n logger.warning(\"Could not re-establish lapsed lease for %s, will retry: %s\", sandbox_id, e)\n return True\n logger.warning(\"Lapsed ownership lease for %s was taken by a peer\", sandbox_id)\n return False\n return False\n\n @contextlib.contextmanager\n def _held_teardown_lease(self, sandbox_id: str):\n \"\"\"Keep *sandbox_id*'s teardown marker alive for as long as its stop runs.\n\n ``claim(..., for_destroy=True)`` writes the ``del:`` marker with the\n ordinary lease TTL, and normal ``renew()`` extends only ``own:`` while\n deliberately reporting a teardown as ``LOST``. Active and unhealthy\n destroy paths drop the sandbox from the maps ``_renew_owned_leases``\n iterates; the warm path keeps its entry visible until the stop succeeds,\n so ``_forget_lost_sandbox`` separately honours ``_local_teardown`` rather\n than misreading our own marker as a peer takeover. Without this heartbeat,\n a container stop that outlived the TTL let the marker lapse, a peer's\n ``take()`` succeeded against the still-running container, and the stop\n landed on the turn that had just been handed it — the very window the\n ``del:`` state exists to close, reopened by its own expiry.\n\n This is what the per-sandbox ``flock`` used to cover for free: a held lock\n cannot expire. A lease can, so the exclusion has to be held deliberately\n rather than assumed to outlast the work it guards. Reachable without an\n abnormal backend — the config schema bounds only ``renewal_interval_seconds``\n (> 0) and ``ttl_multiplier`` (>= 2), so a legal setting puts the TTL below a\n normal container stop, and ``LocalContainerBackend._stop_container`` passes\n no ``timeout`` to ``subprocess.run``, so a wedged daemon blocks unbounded\n even at the default 120s.\n\n The TTL stays finite on purpose: the heartbeat dies with the process, so a\n destroyer that crashes mid-stop still releases the container one TTL later\n instead of marking it undestroyable forever.\n\n The final release is the heartbeat's own last act, not the caller's. A\n refresh ``claim`` still in flight when the context exits (the socket\n timeout bounds it, but it can be mid-call) would otherwise land *after* a\n caller-side release and rewrite the ``del:`` marker on a container whose\n stop had already completed — stranding a fresh ``take()`` (or rolling back\n a fresh create) until the TTL. Releasing from inside the heartbeat, after\n its loop has stopped, sequences the release strictly after the last\n refresh, so no claim can follow it.\n \"\"\"\n stop = threading.Event()\n\n def beat() -> None:\n interval = self._ownership_config.renewal_interval_seconds\n try:\n while not stop.wait(interval):\n try:\n if not self._ownership.claim(sandbox_id, for_destroy=True):\n # Only reachable if the store lost our marker *and* a\n # peer took it (e.g. a flush mid-stop). The stop is\n # already in flight and cannot be recalled, so say so\n # loudly rather than let a peer's container die without\n # a trace.\n logger.error(\n \"Lost the teardown exclusion for %s while its container stop was still in flight; a peer may have taken it\",\n sandbox_id,\n )\n return\n except Exception as e:\n # Broad on purpose: a refresh that raises must not kill the\n # heartbeat and strand the marker for a stop that can run\n # unbounded. Unknown, not lost — the marker may still be\n # live and the TTL bounds a stale one. Retry on the next tick.\n logger.warning(\"Could not refresh the teardown lease for %s, will retry: %s\", sandbox_id, e)\n finally:\n # Release last, from the heartbeat itself, so an in-flight refresh\n # can never run after the marker is cleared. `release()` drops only\n # our own lease, so this is a safe no-op if a peer took it above.\n self._release_ownership(sandbox_id)\n\n beater = threading.Thread(target=beat, name=\"sandbox-teardown-lease\", daemon=True)\n beater.start()\n try:\n yield\n finally:\n stop.set()\n beater.join(timeout=self._TEARDOWN_JOIN_TIMEOUT_SECONDS)\n if beater.is_alive():\n # The budget covers a normally timing-out refresh plus the final\n # release. The release is the heartbeat's job and is still\n # pending; clearing the marker here would reopen the exact race\n # this owns, so leave it — the thread will release when it\n # unblocks, or the TTL will reap it.\n logger.warning(\n \"Teardown heartbeat for %s did not exit within %.1fs; its lease release is deferred to that thread\",\n sandbox_id,\n self._TEARDOWN_JOIN_TIMEOUT_SECONDS,\n )\n\n # ── Startup reconciliation ────────────────────────────────────────────\n\n def _adoptable_after_grace(self, sandbox_id: str, now: float) -> bool:\n \"\"\"Whether *sandbox_id* has looked unowned long enough to be a real orphan.\n\n An absent lease normally proves the owner died and its TTL ran out. But\n the store can lose every key while every owner is alive and serving — a\n Redis restart without persistence, or eviction under ``maxmemory``\n pressure. ``_refresh_ownership`` already refuses to read that as\n abandonment (``LAPSED`` is re-established, not surrendered). Reading the\n same signal as \"orphan, adopt\" here would contradict it on the other\n path: whoever reconciles first would adopt every live container in the\n window before its owner's next renewal tick, that owner's renewal would\n then report ``LOST``, and it would drop a sandbox it is actively serving\n for the adopter to idle-destroy — #4206 through the back door.\n\n Waiting one full lease TTL rebuilds the delay the state loss erased. A\n live owner republishes within one renewal interval, which is shorter than\n the TTL by construction (``ttl_multiplier >= 2``), so only a container\n whose owner is really gone stays unowned across the whole grace.\n \"\"\"\n if not self._ownership.supports_cross_process:\n # No peer can hold a lease this store would show us, so an unowned\n # container cannot be a live peer's — it is from a dead lifecycle of\n # this process. Single-instance deployments keep instant cleanup, and\n # a grace could not help a multi-worker one on this store anyway:\n # peers are invisible to each other's leases with or without it.\n return True\n\n try:\n current_owner = self._ownership.owner(sandbox_id)\n except OwnershipBackendError as e:\n # Unknown, not free: fail closed, same as _claim_ownership.\n logger.warning(\"Could not read sandbox ownership for %s during reconciliation (deferring adoption): %s\", sandbox_id, e)\n return False\n\n if current_owner is not None:\n # Owned — by a peer, or already by us. Either way not an orphan, and\n # a live owner republishing must restart the grace rather than let a\n # stale one expire over its lease.\n self._unowned_since.pop(sandbox_id, None)\n return False\n\n first_seen = self._unowned_since.setdefault(sandbox_id, now)\n return now - first_seen >= compute_lease_ttl(self._ownership_config)\n\n def _reconcile_orphans(self) -> None:\n \"\"\"Reconcile orphaned containers left by previous process lifecycles.\n\n On startup (and periodically from the idle checker), enumerate running\n containers matching our prefix and adopt **true orphans** into the warm\n pool. A container is only adopted when this instance can claim its\n ownership lease — so multi-instance gateways cannot adopt and later\n idle-destroy a peer's live sandbox (#4206).\n\n Adopted orphans get a fresh warm-pool timestamp; the idle checker then\n destroys them if nobody re-acquires within ``idle_timeout``. That still\n cleans containers left by a crashed process once its lease expires.\n\n An unowned container is not adopted on sight — it must stay unowned for a\n recovery grace first, so a store that lost its state cannot be mistaken\n for a fleet of dead owners (see ``_adoptable_after_grace``).\n \"\"\"\n try:\n running = self._backend.list_running()\n except Exception as e:\n logger.warning(f\"Failed to enumerate running containers during startup reconciliation: {e}\")\n return\n\n # Forget grace timers for containers that no longer exist, so a\n # long-lived instance does not accumulate an entry per destroyed\n # container. Runs before the empty-list return so it also drains.\n running_ids = {info.sandbox_id for info in running}\n self._unowned_since = {sid: seen for sid, seen in self._unowned_since.items() if sid in running_ids}\n\n if not running:\n return\n\n current_time = time.time()\n adopted = 0\n skipped_live = 0\n deferred = 0\n\n for info in running:\n age = current_time - info.created_at if info.created_at > 0 else float(\"inf\")\n if not self._adoptable_after_grace(info.sandbox_id, current_time):\n deferred += 1\n logger.debug(\"Deferring container %s during reconciliation: owned, or not yet past the recovery grace\", info.sandbox_id)\n continue\n\n # Claim second: a successful claim proves the container is not a\n # peer's and locks peers out. It says nothing about *us* — it\n # succeeds against our own lease by design — so it is not a substitute\n # for the local teardown check below. The grace above is likewise a\n # precondition, not a substitute; only the claim is atomic.\n if not self._claim_ownership(info.sandbox_id):\n skipped_live += 1\n logger.debug(\"Skipping container %s during reconciliation: owned by another instance\", info.sandbox_id)\n continue\n\n # Single lock acquisition per container: atomic check-and-insert.\n # Avoids a TOCTOU window between the \"already tracked?\" check and the\n # warm-pool insert.\n with self._lock:\n if info.sandbox_id in self._sandboxes or info.sandbox_id in self._warm_pool:\n continue\n if self._being_torn_down_locally(info.sandbox_id):\n # Adoption is a promote, so it needs the same reservation\n # check as the other three. A container being torn down here\n # is untracked and still running, which is exactly the shape\n # this loop adopts — and neither the claim nor the grace\n # excludes it. On `memory` the grace is skipped outright\n # (`supports_cross_process = False`), so nothing else stands\n # in the way there at all; adopting would park a container\n # into the warm pool moments before its stop lands, leaving a\n # dead entry for the next reclaim to hand out.\n deferred += 1\n logger.debug(\"Deferring container %s during reconciliation: this instance is tearing it down\", info.sandbox_id)\n continue\n self._warm_pool[info.sandbox_id] = (info, current_time)\n self._warm_pool_identity[info.sandbox_id] = None\n self._unowned_since.pop(info.sandbox_id, None)\n adopted += 1\n logger.info(f\"Adopted container {info.sandbox_id} into warm pool (age: {age:.0f}s)\")\n\n logger.info(\n \"Startup reconciliation complete: %s adopted into warm pool, %s skipped (live peer ownership), %s deferred (owned or within recovery grace), %s total found\",\n adopted,\n skipped_live,\n deferred,\n len(running),\n )\n\n # ── Deterministic ID ─────────────────────────────────────────────────\n\n @staticmethod\n def _effective_acquire_user_id(user_id: str | None) -> str:\n return user_id or get_effective_user_id()\n\n @staticmethod\n def _thread_key(thread_id: str, user_id: str) -> tuple[str, str]:\n return (user_id, thread_id)\n\n @staticmethod\n def _deterministic_sandbox_id(thread_id: str, user_id: str) -> str:\n \"\"\"Generate a deterministic sandbox ID from user/thread scope.\n\n Includes user_id so a previously-created default-bucket sandbox cannot be\n reused for an auth/channel run that should mount a user-scoped bucket.\n\n During a mixed-version rollout, older 8-character containers are not\n reused under the new 16-character identity. They remain eligible for\n normal orphan cleanup while the first new-version acquire cold-starts.\n \"\"\"\n return hashlib.sha256(f\"{user_id}:{thread_id}\".encode()).hexdigest()[:16]\n\n def _assert_active_identity_available_locked(\n self,\n sandbox_id: str,\n requested_key: tuple[str, str],\n ) -> None:\n \"\"\"Fail closed if an active truncated ID belongs to another identity.\"\"\"\n if sandbox_id not in self._sandboxes and sandbox_id not in self._sandbox_infos:\n return\n\n stored_key = self._active_sandbox_identity.get(sandbox_id)\n if stored_key is None:\n matching_keys = [key for key, mapped_id in self._thread_sandboxes.items() if mapped_id == sandbox_id]\n if len(matching_keys) == 1:\n stored_key = matching_keys[0]\n if stored_key != requested_key:\n raise SandboxIdentityCollisionError(sandbox_id, stored_key, requested_key)\n\n def _assert_warm_identity_available_locked(\n self,\n sandbox_id: str,\n requested_key: tuple[str, str],\n ) -> None:\n \"\"\"Fail closed if a warm ID changed tenants during an acquire.\"\"\"\n if sandbox_id not in self._warm_pool:\n return\n # Startup-adopted entries have unknown identity until their first reclaim.\n stored_key = self._warm_pool_identity.get(sandbox_id)\n if stored_key is not None and stored_key != requested_key:\n raise SandboxIdentityCollisionError(sandbox_id, stored_key, requested_key)\n\n # ── Mount helpers ────────────────────────────────────────────────────\n\n def _get_extra_mounts(self, thread_id: str | None, *, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Collect all extra mounts for a sandbox (thread-specific + skills).\"\"\"\n mounts: list[tuple[str, str, bool]] = []\n\n if thread_id:\n mounts.extend(self._get_thread_mounts(thread_id, user_id=user_id))\n logger.info(f\"Adding thread mounts for thread {thread_id}: {mounts}\")\n\n skills_mounts = self._get_skills_mounts(user_id=user_id)\n if skills_mounts:\n mounts.extend(skills_mounts)\n logger.info(f\"Adding skills mounts: {skills_mounts}\")\n\n user_skill_mounts = self._get_user_skill_mounts(user_id=user_id)\n if user_skill_mounts:\n mounts.extend(user_skill_mounts)\n logger.info(f\"Adding user skill mounts: {user_skill_mounts}\")\n\n lark_cli_mounts = self._get_lark_cli_runtime_mounts(user_id=user_id)\n if lark_cli_mounts:\n mounts.extend(lark_cli_mounts)\n logger.info(f\"Adding Lark CLI runtime mounts: {lark_cli_mounts}\")\n\n return self._dedupe_mounts_by_container_path(mounts)\n\n @staticmethod\n def _dedupe_mounts_by_container_path(mounts: list[tuple[str, str, bool]]) -> list[tuple[str, str, bool]]:\n \"\"\"Keep the first mount for each container path.\n\n Duplicate container paths are rejected by the provisioner and can also\n fail local Docker creation. The earlier mount wins because mount helpers\n are appended in priority order: thread data, skill roots, integration\n skill roots, then integration runtimes/credentials.\n \"\"\"\n seen: set[str] = set()\n deduped: list[tuple[str, str, bool]] = []\n for host_path, container_path, read_only in mounts:\n if container_path in seen:\n logger.warning(\n \"Skipping duplicate sandbox mount for container path %s from host %s\",\n container_path,\n host_path,\n )\n continue\n seen.add(container_path)\n deduped.append((host_path, container_path, read_only))\n return deduped\n\n @staticmethod\n def _get_thread_mounts(thread_id: str, *, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Get volume mounts for a thread's data directories.\n\n Creates directories if they don't exist (lazy initialization).\n Mount sources use host_base_dir so that when running inside Docker with a\n mounted Docker socket (DooD), the host Docker daemon can resolve the paths.\n \"\"\"\n paths = get_paths()\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n paths.ensure_thread_dirs(thread_id, user_id=effective_user_id)\n\n return [\n (paths.host_sandbox_work_dir(thread_id, user_id=effective_user_id), f\"{VIRTUAL_PATH_PREFIX}/workspace\", False),\n (paths.host_sandbox_uploads_dir(thread_id, user_id=effective_user_id), f\"{VIRTUAL_PATH_PREFIX}/uploads\", False),\n (paths.host_sandbox_outputs_dir(thread_id, user_id=effective_user_id), f\"{VIRTUAL_PATH_PREFIX}/outputs\", False),\n # ACP workspace: read-only inside the sandbox (lead agent reads results;\n # the ACP subprocess writes from the host side, not from within the container).\n (paths.host_acp_workspace_dir(thread_id, user_id=effective_user_id), \"/mnt/acp-workspace\", True),\n ]\n\n @staticmethod\n def _get_skills_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Get skills directory mount configurations for three-way skills layout.\n\n Mirrors ``LocalSandboxProvider._build_thread_path_mappings`` for AIO\n sandboxes: public, per-user custom, and legacy (pre-migration\n global-custom) skills are mounted to separate container subdirectories so\n that ``Skill.get_container_path()`` category-aware paths resolve\n correctly inside the sandbox.\n\n Mount sources use ``DEER_FLOW_HOST_BASE_DIR`` when running inside\n Docker (DooD) so the host Docker daemon can resolve the projection\n paths.\n \"\"\"\n mounts: list[tuple[str, str, bool]] = []\n try:\n config = get_app_config()\n container_path = config.skills.container_path\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n AioSandboxProvider._ensure_skills_projection(effective_user_id)\n paths = get_paths()\n host_base_dir = str(paths.host_base_dir)\n\n # 1. Public skills: global, read-only — static, shared by all threads\n mounts.append(\n (\n join_host_path(host_base_dir, \"skills_view\", \"public\"),\n f\"{container_path}/public\",\n True,\n )\n )\n\n # 2. Per-user custom skills: read-only, per-thread/per-user\n host_user_custom = join_host_path(\n host_base_dir,\n \"users\",\n effective_user_id,\n \"skills_view\",\n \"custom\",\n )\n mounts.append(\n (\n host_user_custom,\n f\"{container_path}/custom\",\n True,\n )\n )\n\n # 3. Legacy visibility is encoded by projection contents. Keep the\n # mount stable even when the directory is empty so a later state\n # change is visible without recreating the sandbox.\n mounts.append(\n (\n join_host_path(host_base_dir, \"users\", effective_user_id, \"skills_view\", \"legacy\"),\n f\"{container_path}/legacy\",\n True,\n )\n )\n except Exception as e:\n logger.warning(\"Could not setup skills mounts: %s\", e)\n\n return mounts\n\n @staticmethod\n def _ensure_skills_projection(user_id: str):\n \"\"\"Best-effort: a projection failure must not fail sandbox acquire.\n\n Called directly (for its side effect) from ``_acquire_internal`` /\n ``_acquire_internal_async`` outside any try/except, as well as from\n within ``_get_skills_mounts``'s own guarded block — swallowing here\n keeps both call sites safe without duplicating the guard.\n \"\"\"\n from deerflow.skills.projection import ensure_skill_projections\n from deerflow.skills.storage import get_or_new_user_skill_storage\n\n try:\n storage = get_or_new_user_skill_storage(user_id, app_config=get_app_config())\n return ensure_skill_projections(storage)\n except Exception as exc:\n logger.warning(\"Could not ensure skills projection for user %s: %s\", user_id, exc, exc_info=True)\n return None\n\n @staticmethod\n def _get_user_skill_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Mount enabled managed integration skills into AIO sandboxes.\n\n Per-user custom skills are already mounted by ``_get_skills_mounts``.\n Integration packages are shared, but their enabled state is per-user, so\n this helper mounts the user's projection instead of the raw shared root.\n \"\"\"\n try:\n config = get_app_config()\n paths = get_paths()\n skills_container_path = config.skills.container_path\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n AioSandboxProvider._ensure_skills_projection(effective_user_id)\n return [\n (\n join_host_path(\n str(paths.host_base_dir),\n \"users\",\n effective_user_id,\n \"skills_view\",\n \"integrations\",\n ),\n f\"{skills_container_path}/integrations\",\n True,\n ),\n ]\n except Exception as e:\n logger.warning(f\"Could not setup user skill mounts: {e}\")\n return []\n\n @staticmethod\n def _lark_integration_active(user_id: str | None = None) -> bool:\n \"\"\"Whether the managed Lark skill pack is installed for this user.\n\n Drives whether a sandbox requests the lark-cli runtime (init container /\n Gateway-download mount). Independent of whether a local ``sandbox-cli``\n dir exists, so remote/K8s can opt in without a Gateway-side download.\n \"\"\"\n try:\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n return lark_skills_installed(effective_user_id)\n except Exception as e: # pragma: no cover - defensive\n logger.warning(f\"Could not determine Lark integration state: {e}\")\n return False\n\n @staticmethod\n def _lark_broker_active(user_id: str | None = None) -> bool:\n \"\"\"Whether this user's sandbox should use the lark-cli broker (Pattern B).\n\n True only when the Lark pack is installed AND the remote provisioner\n reports a configured broker image. When true, the provisioner keeps the\n credentials in a sidecar and the sandbox gets only a shim, so the\n Gateway-side credential-mount overlay must not run either.\n \"\"\"\n try:\n if not AioSandboxProvider._lark_integration_active(user_id):\n return False\n from deerflow.integrations.lark_cli import sandbox_lark_broker_active\n\n return sandbox_lark_broker_active()\n except Exception as e: # pragma: no cover - defensive\n logger.warning(f\"Could not determine Lark broker state: {e}\")\n return False\n\n @staticmethod\n def _get_lark_cli_runtime_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]:\n \"\"\"Mount the per-user lark-cli config/data dirs used by Settings auth.\n\n Settings endpoints run ``lark-cli`` on the Gateway with\n ``LARKSUITE_CLI_CONFIG_DIR`` / ``DATA_DIR`` pointing at\n ``users/{user}/integrations/lark-cli``. Agent conversations run\n ``lark-cli`` inside the sandbox, so those same directories must be\n mounted into the container or the CLI sees a separate unauthenticated\n profile.\n\n The ``config`` dir holds the long-lived Lark ``appSecret`` (written by\n ``lark-cli config init`` on the Gateway, never in-sandbox), so it is\n mounted **read-only**: sandbox processes only need to read it, and a\n read-only bind stops a compromised agent from tampering with or\n replacing the app credentials. The ``data`` dir holds refreshable OAuth\n tokens that ``lark-cli auth`` updates in-sandbox, so it stays writable.\n This is defense-in-depth only — both dirs remain readable to arbitrary\n sandbox processes until the auth-proxy follow-up (issue #4338) lands.\n See the sandbox trust-boundary note in ``backend/AGENTS.md``.\n \"\"\"\n try:\n paths = get_paths()\n effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)\n ensure_lark_cli_credential_tree(effective_user_id, paths=paths)\n mounts = [\n (paths.host_user_integration_config_dir(effective_user_id, LARK_CLI_INTEGRATION_ID), LARK_CLI_SANDBOX_CONFIG_DIR, True),\n (paths.host_user_integration_data_dir(effective_user_id, LARK_CLI_INTEGRATION_ID), LARK_CLI_SANDBOX_DATA_DIR, False),\n ]\n runtime_dir = paths.base_dir / \"integrations\" / LARK_CLI_INTEGRATION_ID / \"sandbox-cli\"\n if runtime_dir.is_dir():\n mounts.append(\n (\n join_host_path(str(paths.host_base_dir), \"integrations\", LARK_CLI_INTEGRATION_ID, \"sandbox-cli\"),\n LARK_CLI_SANDBOX_RUNTIME_DIR,\n True,\n )\n )\n return mounts\n except Exception as e:\n logger.warning(f\"Could not setup Lark CLI runtime mounts: {e}\")\n return []\n\n # ── Idle timeout management ──────────────────────────────────────────\n\n def _cleanup_idle_resources(self, idle_timeout: float) -> None:\n \"\"\"Clean AIO resources idle longer than ``idle_timeout`` seconds.\"\"\"\n # Pick up containers whose peer leases expired since startup (crash path).\n self._reconcile_orphans()\n self._cleanup_idle_sandboxes(idle_timeout)\n\n # ── Ownership lease renewal ──────────────────────────────────────────\n\n def _start_lease_renewal(self) -> None:\n \"\"\"Start the daemon thread that keeps this instance's leases alive.\n\n Deliberately not folded into the idle checker: that thread only starts\n when ``idle_timeout > 0``, so renewal riding on it silently stopped for\n ``idle_timeout: 0`` deployments — a supported config (\"keep warm VMs\n until shutdown\") — letting every lease lapse and reopening #4206 one TTL\n later. Liveness and reaping must not share a switch.\n \"\"\"\n if self._renewal_thread is not None and self._renewal_thread.is_alive():\n return\n\n self._renewal_stop.clear()\n self._renewal_thread = threading.Thread(\n target=self._lease_renewal_loop,\n name=\"sandbox-lease-renewal\",\n daemon=True,\n )\n self._renewal_thread.start()\n logger.info(\n \"Started sandbox ownership renewal thread (interval: %.1fs, ttl: %.1fs)\",\n self._ownership_config.renewal_interval_seconds,\n self._ownership_config.renewal_interval_seconds * self._ownership_config.ttl_multiplier,\n )\n\n def _stop_lease_renewal(self) -> None:\n self._renewal_stop.set()\n thread = self._renewal_thread\n if thread is not None and thread.is_alive() and thread is not threading.current_thread():\n thread.join(timeout=5)\n\n def _lease_renewal_loop(self) -> None:\n interval = self._ownership_config.renewal_interval_seconds\n while not self._renewal_stop.wait(interval):\n try:\n self._renew_owned_leases()\n except Exception:\n logger.exception(\"Error in sandbox ownership renewal loop\")\n\n def _renew_owned_leases(self) -> None:\n \"\"\"Renew every container this instance believes it owns.\n\n Covers warm entries as well as active ones: a warm container is still\n ours (we hold it for fast reclaim), so letting its lease lapse would let\n a peer adopt a container we are about to hand back to its thread.\n\n Only a lease a **peer** now holds means the container is no longer ours;\n a lapsed one is re-established (see ``_refresh_ownership``). Conflating\n the two would evict every live sandbox on this instance the first time\n the store lost its state.\n \"\"\"\n with self._lock:\n owned_ids = list(self._sandboxes.keys()) + list(self._warm_pool.keys())\n\n for sandbox_id in owned_ids:\n # Snapshot before the round trip: by the time `renew()` answers LOST,\n # an acquire in this process may already have taken the lease back\n # and promoted the id, and the answer is about the lease we held then.\n epoch = self._acquire_epoch_of(sandbox_id)\n if not self._refresh_ownership(sandbox_id):\n logger.warning(\"Lost sandbox ownership lease for %s; dropping it from this instance\", sandbox_id)\n self._forget_lost_sandbox(sandbox_id, expected_epoch=epoch)\n\n def _forget_lost_sandbox(self, sandbox_id: str, *, expected_epoch: int | None = None) -> None:\n \"\"\"Drop a sandbox whose lease we no longer hold, without touching the container.\n\n The container now belongs to whichever instance holds the lease, so\n stopping it here would be the very cross-instance kill this store exists\n to prevent. Only our host-side handle goes away.\n\n ``expected_epoch`` guards callers whose \"we lost it\" decision came from a\n store round trip made outside the lock. An acquire **mid-flight** counts\n too: its ``take()`` can already have made the takeover durable while the\n epoch is still unwritten, so the epoch alone would let a stale decision\n through (see ``_publish_ownership``). An acquire that re-took the lease\n in that window has already handed the sandbox to a turn — and, on the\n reuse path, handed out the *same* tracked client, so no object-identity\n check would notice. Dropping it then closes a client mid-turn and leaves\n the agent holding an id whose tool calls fail until the next turn.\n \"\"\"\n with self._lock:\n # A warm teardown deliberately keeps its entry visible until the\n # backend stop succeeds. Its own `del:` marker makes `renew()` report\n # LOST, but that is not a peer takeover and must not pop the retained\n # entry — especially when the stop fails and the container remains\n # live for retry/reclaim. The teardown path removes it on success.\n if sandbox_id in self._local_teardown:\n logger.debug(\"Not dropping sandbox %s: this instance is tearing it down\", sandbox_id)\n return\n # The in-flight check is deliberately *not* conditional on\n # `expected_epoch`. Today's epoch-less callers (the two\n # `SandboxBeingDestroyedError` handlers) cannot collide with a\n # publish for the same id — `_publish_ownership` has already cleared\n # the mark by the time they run, and acquires for one id are\n # serialized by the per-thread lock — so this changes no current\n # behaviour. It is here because \"no epoch supplied\" reading as \"no\n # guard at all\" is how the next caller of a dangerous primitive gets\n # written; an id being acquired right now must never be dropped.\n if sandbox_id in self._acquire_inflight:\n logger.info(\"Not dropping sandbox %s: an acquire is publishing ownership for it\", sandbox_id)\n return\n if expected_epoch is not None and self._acquire_epoch.get(sandbox_id, 0) != expected_epoch:\n logger.info(\"Not dropping sandbox %s: this instance re-acquired it after the lease check\", sandbox_id)\n return\n\n sandbox = self._sandboxes.pop(sandbox_id, None)\n self._sandbox_infos.pop(sandbox_id, None)\n self._active_sandbox_identity.pop(sandbox_id, None)\n self._last_activity.pop(sandbox_id, None)\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n self._acquire_epoch.pop(sandbox_id, None)\n for key, mapped_id in list(self._thread_sandboxes.items()):\n if mapped_id == sandbox_id:\n del self._thread_sandboxes[key]\n\n # Close the host-side HTTP client we are dropping (#2872); the container\n # itself stays up for its new owner.\n if sandbox is not None:\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} after losing its lease: {e}\")\n\n def _cleanup_idle_sandboxes(self, idle_timeout: float) -> None:\n current_time = time.time()\n active_to_destroy = []\n\n with self._lock:\n # Active sandboxes: tracked via _last_activity\n for sandbox_id, last_activity in self._last_activity.items():\n idle_duration = current_time - last_activity\n if idle_duration > idle_timeout:\n active_to_destroy.append(sandbox_id)\n logger.info(f\"Sandbox {sandbox_id} idle for {idle_duration:.1f}s, marking for destroy\")\n\n # Destroy active sandboxes (re-verify still idle before acting).\n #\n # The re-verify has to happen in the same critical section as the\n # teardown reservation, which is why it is handed to `_destroy_tracked`\n # as a predicate rather than run here. Checking here and destroying\n # afterwards left a window — widened by this PR from a few instructions\n # to a store round trip, since `destroy()` now claims ownership before it\n # untracks — in which a turn re-acquires the sandbox and then has its\n # container stopped underneath it.\n def still_idle(sandbox_id: str) -> bool:\n last_activity = self._last_activity.get(sandbox_id)\n if last_activity is None:\n # Already released or destroyed by another path — skip.\n logger.info(f\"Sandbox {sandbox_id} already gone before idle destroy, skipping\")\n return False\n if (time.time() - last_activity) < idle_timeout:\n # Re-acquired (activity updated) since the snapshot — skip.\n logger.info(f\"Sandbox {sandbox_id} was re-acquired before idle destroy, skipping\")\n return False\n return True\n\n for sandbox_id in active_to_destroy:\n try:\n logger.info(f\"Destroying idle sandbox {sandbox_id}\")\n self._destroy_tracked(sandbox_id, still_reapable=lambda sid=sandbox_id: still_idle(sid))\n except Exception as e:\n logger.error(f\"Failed to destroy idle sandbox {sandbox_id}: {e}\")\n\n self._reap_expired_warm(idle_timeout)\n\n def _reap_expired_warm(self, idle_timeout: float | None = None) -> None:\n \"\"\"Destroy warm entries older than ``idle_timeout``, never a peer's live container.\"\"\"\n timeout = float(self._config.get(\"idle_timeout\", DEFAULT_IDLE_TIMEOUT) if idle_timeout is None else idle_timeout)\n if timeout <= 0:\n return\n\n now = time.time()\n expired: list[tuple[str, SandboxInfo]] = []\n with self._lock:\n for sandbox_id, (entry, timestamp) in self._warm_pool.items():\n if now - timestamp > timeout:\n expired.append((sandbox_id, entry))\n\n # Only drop an entry from the warm pool once we know it is really going\n # away. Popping first would lose the container on a refused or\n # unanswerable claim: still running, no longer tracked by anyone. The\n # deferred pop is why the reservation is needed — the entry stays visible\n # to `_reclaim_warm_pool_sandbox` for the whole stop.\n for sandbox_id, entry in expired:\n self._destroy_warm_entry(sandbox_id, entry, reason=\"idle_timeout\", still_reapable=lambda sid=sandbox_id: sid in self._warm_pool)\n\n def _evict_oldest_warm(self) -> str | None:\n \"\"\"Evict the oldest warm entry this instance still owns.\"\"\"\n with self._lock:\n if not self._warm_pool:\n return None\n # Snapshot oldest-first under the lock; ownership is resolved outside\n # it, since a claim can be a network round trip and the provider lock\n # guards every acquire path.\n candidates = [(sandbox_id, entry) for sandbox_id, (entry, _) in sorted(self._warm_pool.items(), key=lambda item: item[1][1])]\n\n for sandbox_id, entry in candidates:\n # \"Still in the warm pool?\" is the reapable check, and it has to run\n # in the same critical section as the reservation — checking it here\n # and reserving afterwards is exactly the window a reclaim slips\n # through. `_destroy_warm_entry` does both under one lock hold.\n if not self._destroy_warm_entry(sandbox_id, entry, reason=\"replica_enforcement\", still_reapable=lambda sid=sandbox_id: sid in self._warm_pool):\n continue\n return sandbox_id\n\n return None\n\n # ── Signal handling ──────────────────────────────────────────────────\n\n def _register_signal_handlers(self) -> None:\n \"\"\"Register signal handlers for graceful shutdown.\n\n Handles SIGTERM, SIGINT, and SIGHUP (terminal close) to ensure\n sandbox containers are cleaned up even when the user closes the terminal.\n \"\"\"\n self._original_sigterm = signal.getsignal(signal.SIGTERM)\n self._original_sigint = signal.getsignal(signal.SIGINT)\n self._original_sighup = signal.getsignal(signal.SIGHUP) if hasattr(signal, \"SIGHUP\") else None\n\n def signal_handler(signum, frame):\n self.shutdown()\n if signum == signal.SIGTERM:\n original = self._original_sigterm\n elif hasattr(signal, \"SIGHUP\") and signum == signal.SIGHUP:\n original = self._original_sighup\n else:\n original = self._original_sigint\n if callable(original):\n original(signum, frame)\n elif original == signal.SIG_DFL:\n signal.signal(signum, signal.SIG_DFL)\n signal.raise_signal(signum)\n\n try:\n signal.signal(signal.SIGTERM, signal_handler)\n signal.signal(signal.SIGINT, signal_handler)\n if hasattr(signal, \"SIGHUP\"):\n signal.signal(signal.SIGHUP, signal_handler)\n except ValueError:\n logger.debug(\"Could not register signal handlers (not main thread)\")\n\n # ── Thread locking (in-process) ──────────────────────────────────────\n\n def _get_thread_lock(self, thread_id: str, user_id: str) -> threading.Lock:\n \"\"\"Get or create an in-process lock for a specific user/thread scope.\"\"\"\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n if key not in self._thread_locks:\n self._thread_locks[key] = threading.Lock()\n return self._thread_locks[key]\n\n def _sandbox_id_for_thread(self, thread_id: str | None, user_id: str | None) -> str:\n \"\"\"Return deterministic IDs for thread sandboxes and random IDs otherwise.\"\"\"\n return self._deterministic_sandbox_id(thread_id, self._effective_acquire_user_id(user_id)) if thread_id else str(uuid.uuid4())[:8]\n\n def _reuse_in_process_sandbox(self, thread_id: str | None, *, user_id: str | None = None, post_lock: bool = False) -> str | None:\n \"\"\"Reuse an active in-process sandbox for a thread if one is still tracked.\"\"\"\n if thread_id is None:\n return None\n\n effective_user_id = self._effective_acquire_user_id(user_id)\n key = self._thread_key(thread_id, effective_user_id)\n with self._lock:\n if key not in self._thread_sandboxes:\n return None\n\n existing_id = self._thread_sandboxes[key]\n if self._being_torn_down_locally(existing_id):\n # A reaper thread in this process is stopping this container.\n # Same answer as a peer's `del:` lease: cold-start instead.\n logger.info(\"Cached sandbox %s is being destroyed by this instance; not reusing it\", existing_id)\n return None\n if existing_id in self._sandboxes:\n info = self._sandbox_infos.get(existing_id)\n else:\n del self._thread_sandboxes[key]\n return None\n\n alive = self._check_tracked_sandbox_alive(existing_id, info) if info is not None else True\n if alive is False:\n self._drop_unhealthy_sandbox(\n existing_id,\n \"in-process cache failed health check\",\n expected_info=info,\n )\n return None\n\n with self._lock:\n if self._thread_sandboxes.get(key) != existing_id:\n return None\n if existing_id not in self._sandboxes:\n self._thread_sandboxes.pop(key, None)\n return None\n\n suffix = \" (post-lock check)\" if post_lock else \"\"\n logger.info(f\"Reusing in-process sandbox {existing_id} for user/thread {effective_user_id}/{thread_id}{suffix}\")\n self._last_activity[existing_id] = time.time()\n\n # Fail closed: an OwnershipBackendError propagates rather than handing out\n # a sandbox we could not publish ownership for.\n try:\n self._publish_ownership(existing_id)\n except SandboxBeingDestroyedError:\n # A peer is stopping this container. Drop it and let the caller\n # discover-or-create a fresh one instead of handing over a sandbox\n # that is about to disappear.\n logger.info(\"Cached sandbox %s is being destroyed by another instance; not reusing it\", existing_id)\n self._forget_lost_sandbox(existing_id)\n return None\n\n with self._lock:\n if self._being_torn_down_locally(existing_id):\n # The first reservation check ran before the backend health\n # check and ownership round trip. A local reaper can win while\n # either is in flight, and it deliberately keeps the entry in\n # `_sandboxes` until its destroy claim succeeds. Membership\n # alone therefore cannot prove this id is still safe to return.\n logger.info(\"Cached sandbox %s was reserved for teardown while publishing ownership; not reusing it\", existing_id)\n return None\n if existing_id not in self._sandboxes:\n # Dropped while we were publishing. The intent mark closes the\n # window *inside* `_publish_ownership`, but not the gap before\n # it: until the mark is set a renewal's `LOST` is both current\n # and correct — the peer really did hold the lease — so the\n # forget legitimately runs and closes this client. Returning the\n # id anyway would hand back a sandbox whose `get()` is `None`.\n # Fall through instead; the caller re-discovers and builds a\n # fresh client, and the lease we just took is already ours.\n logger.info(\"Cached sandbox %s was dropped while publishing ownership; falling through to discovery\", existing_id)\n return None\n return existing_id\n\n def _reclaim_warm_pool_sandbox(\n self,\n thread_id: str | None,\n sandbox_id: str,\n *,\n user_id: str | None = None,\n post_lock: bool = False,\n ) -> str | None:\n \"\"\"Promote a warm-pool sandbox back to active tracking if available.\"\"\"\n if thread_id is None:\n return None\n\n effective_user_id = self._effective_acquire_user_id(user_id)\n key = self._thread_key(thread_id, effective_user_id)\n with self._lock:\n if sandbox_id not in self._warm_pool:\n return None\n self._assert_warm_identity_available_locked(sandbox_id, key)\n if self._being_torn_down_locally(sandbox_id):\n # The entry deliberately stays in `_warm_pool` for the whole stop\n # (so a refused claim does not lose the container), so pool\n # membership alone does not mean it is reclaimable.\n logger.info(\"Warm-pool sandbox %s is being destroyed by this instance; not reclaiming it\", sandbox_id)\n return None\n\n info, _ = self._warm_pool[sandbox_id]\n\n alive = self._check_tracked_sandbox_alive(sandbox_id, info)\n if alive is False:\n self._drop_unhealthy_sandbox(\n sandbox_id,\n \"warm-pool cache failed health check\",\n expected_info=info,\n )\n return None\n\n # Publish ownership before the warm → active transition: a raise here must\n # not leave the sandbox tracked as active but unowned (a peer would see an\n # orphan and reap it mid-turn). On failure the entry stays warm and this\n # instance keeps its existing lease.\n try:\n self._publish_ownership(sandbox_id)\n except SandboxBeingDestroyedError:\n logger.info(\"Warm-pool sandbox %s is being destroyed by another instance; not reclaiming it\", sandbox_id)\n self._forget_lost_sandbox(sandbox_id)\n return None\n\n with self._lock:\n if self._being_torn_down_locally(sandbox_id):\n # Re-checked, because the first check was before the round trip.\n # A reaper can reserve *after* our `take()` — the warm entry is\n # still there, since its pop is deferred until the stop returns —\n # then claim `del:` (which succeeds: the lease is ours, we just\n # took it) and stop the container. Whichever pop lands first\n # decides, and if ours does we install a client for a container\n # that is already stopped.\n logger.info(\"Warm-pool sandbox %s was claimed for teardown while publishing ownership; not reclaiming it\", sandbox_id)\n return None\n self._assert_warm_identity_available_locked(sandbox_id, key)\n warm_item = self._warm_pool.pop(sandbox_id, None)\n if warm_item is None:\n return None\n self._warm_pool_identity.pop(sandbox_id, None)\n info, _ = warm_item\n sandbox = AioSandbox(id=sandbox_id, base_url=info.sandbox_url)\n self._sandboxes[sandbox_id] = sandbox\n self._sandbox_infos[sandbox_id] = info\n self._active_sandbox_identity[sandbox_id] = key\n self._last_activity[sandbox_id] = time.time()\n self._thread_sandboxes[key] = sandbox_id\n\n suffix = \" (post-lock check)\" if post_lock else f\" at {info.sandbox_url}\"\n logger.info(f\"Reclaimed warm-pool sandbox {sandbox_id} for user/thread {effective_user_id}/{thread_id}{suffix}\")\n return sandbox_id\n\n def _recheck_cached_sandbox(self, thread_id: str, sandbox_id: str, *, user_id: str) -> str | None:\n \"\"\"Re-check in-memory caches after acquiring the cross-process file lock.\"\"\"\n return self._reuse_in_process_sandbox(thread_id, user_id=user_id, post_lock=True) or self._reclaim_warm_pool_sandbox(\n thread_id,\n sandbox_id,\n user_id=user_id,\n post_lock=True,\n )\n\n def _register_discovered_sandbox(self, thread_id: str, info: SandboxInfo, *, user_id: str) -> str:\n \"\"\"Track a sandbox discovered through the backend.\n\n Raises:\n SandboxBeingDestroyedError: discovery found the container still\n running, but a peer is stopping it. Deliberately propagated\n rather than swallowed: falling through to create would collide\n with the not-yet-removed container name, and handing this one to\n an agent is exactly the mid-turn death (#4206) the store exists to\n prevent. The window is a peer's in-flight container stop, so the\n thread's next turn discovers nothing and cold-starts cleanly.\n \"\"\"\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n if self._being_torn_down_locally(info.sandbox_id):\n # Discovery is the fall-through once the caches miss, so it is\n # also the path a reaper's own untracking opens up. `take()` would\n # only refuse this once the reaper's `del:` claim has landed;\n # until then it succeeds against our own lease.\n raise SandboxBeingDestroyedError(info.sandbox_id)\n self._assert_active_identity_available_locked(info.sandbox_id, key)\n self._assert_warm_identity_available_locked(info.sandbox_id, key)\n\n sandbox = AioSandbox(id=info.sandbox_id, base_url=info.sandbox_url)\n # Ownership first, so a failure cannot leave a tracked-but-unowned sandbox.\n # There is no container to roll back (we did not create it), but the\n # host-side HTTP client constructed above is ours and must not leak —\n # same close-on-failure as `_register_created_sandbox`.\n try:\n self._publish_ownership(info.sandbox_id)\n with self._lock:\n if self._being_torn_down_locally(info.sandbox_id):\n # The pre-publish reservation check is only an early-out: a\n # local reaper can reserve the id during the store round\n # trip. Do not install a client for a container that reaper\n # has already committed to stopping.\n raise SandboxBeingDestroyedError(info.sandbox_id)\n self._assert_active_identity_available_locked(info.sandbox_id, key)\n self._assert_warm_identity_available_locked(info.sandbox_id, key)\n # Active and warm are exclusive states, and only this insert can\n # violate that: a warm entry for the same id is stale the moment\n # the id becomes active. Leaving it there gives the container two\n # reapers — `_reap_expired_warm` judges it by the warm timestamp\n # and never looks at `_last_activity`, so it stops a container an\n # agent is actively using while `_sandboxes` still hands out its\n # client.\n self._warm_pool.pop(info.sandbox_id, None)\n self._warm_pool_identity.pop(info.sandbox_id, None)\n self._sandboxes[info.sandbox_id] = sandbox\n self._sandbox_infos[info.sandbox_id] = info\n self._active_sandbox_identity[info.sandbox_id] = key\n self._last_activity[info.sandbox_id] = time.time()\n self._thread_sandboxes[key] = info.sandbox_id\n except (\n OwnershipBackendError,\n SandboxBeingDestroyedError,\n SandboxIdentityCollisionError,\n ):\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {info.sandbox_id} after failed ownership publish: {e}\")\n raise\n\n logger.info(f\"Discovered existing sandbox {info.sandbox_id} for user/thread {user_id}/{thread_id} at {info.sandbox_url}\")\n return info.sandbox_id\n\n def _register_created_sandbox(self, thread_id: str | None, sandbox_id: str, info: SandboxInfo, *, user_id: str | None = None) -> str:\n \"\"\"Track a newly-created sandbox in the active maps.\"\"\"\n sandbox = AioSandbox(id=sandbox_id, base_url=info.sandbox_url)\n key = (\n self._thread_key(\n thread_id,\n self._effective_acquire_user_id(user_id),\n )\n if thread_id\n else None\n )\n # Ownership first. Unlike the discover path there IS something to roll\n # back: we just started this container, and an unowned running container\n # is exactly what a peer's reconciliation adopts. Leaking it would hand a\n # peer a container this instance is about to use.\n # SandboxBeingDestroyedError is possible even here: a peer that died\n # mid-stop leaves a teardown marker until its TTL lapses. Roll back on\n # both, or the container we just started is leaked.\n try:\n if key is not None:\n with self._lock:\n self._assert_active_identity_available_locked(sandbox_id, key)\n self._assert_warm_identity_available_locked(sandbox_id, key)\n self._publish_ownership(sandbox_id)\n\n with self._lock:\n if key is not None:\n self._assert_active_identity_available_locked(sandbox_id, key)\n self._assert_warm_identity_available_locked(sandbox_id, key)\n # Same exclusivity rule as the discover path.\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n self._sandboxes[sandbox_id] = sandbox\n self._sandbox_infos[sandbox_id] = info\n self._active_sandbox_identity[sandbox_id] = key\n self._last_activity[sandbox_id] = time.time()\n if key is not None:\n self._thread_sandboxes[key] = sandbox_id\n except (\n OwnershipBackendError,\n SandboxBeingDestroyedError,\n SandboxIdentityCollisionError,\n ):\n logger.error(\n \"Could not register new sandbox %s; destroying it rather than leaking an untracked container\",\n sandbox_id,\n )\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} during ownership rollback: {e}\")\n try:\n self._backend.destroy(info)\n except Exception as e:\n logger.error(\n \"Failed to destroy sandbox %s after registration failure: %s\",\n sandbox_id,\n e,\n )\n raise\n\n logger.info(f\"Created sandbox {sandbox_id} for thread {thread_id} at {info.sandbox_url}\")\n return sandbox_id\n\n def _check_tracked_sandbox_alive(self, sandbox_id: str, info: SandboxInfo) -> bool | None:\n \"\"\"Return whether a tracked sandbox appears alive, or None if unknown.\"\"\"\n try:\n return self._backend.is_alive(info)\n except Exception as e:\n logger.warning(f\"Failed to check sandbox {sandbox_id} health: {e}\")\n return None\n\n def _remove_tracked_sandbox(\n self,\n sandbox_id: str,\n *,\n expected_info: SandboxInfo | None = None,\n ) -> tuple[Sandbox | None, SandboxInfo | None, bool]:\n \"\"\"Remove a sandbox from in-process tracking maps.\n\n When expected_info is provided, removal only happens if the currently\n tracked active or warm-pool entry is the exact info object that was\n checked. This prevents a stale health-check result from deleting a\n freshly recreated sandbox with the same deterministic id.\n \"\"\"\n thread_keys_to_remove: list[tuple[str, str]] = []\n\n with self._lock:\n active_info = self._sandbox_infos.get(sandbox_id)\n warm_item = self._warm_pool.get(sandbox_id)\n warm_info = warm_item[0] if warm_item is not None else None\n if expected_info is not None and active_info is not expected_info and warm_info is not expected_info:\n return None, None, False\n\n sandbox = self._sandboxes.pop(sandbox_id, None)\n info = self._sandbox_infos.pop(sandbox_id, None)\n self._active_sandbox_identity.pop(sandbox_id, None)\n thread_keys_to_remove = [key for key, sid in self._thread_sandboxes.items() if sid == sandbox_id]\n for key in thread_keys_to_remove:\n del self._thread_sandboxes[key]\n self._last_activity.pop(sandbox_id, None)\n self._acquire_epoch.pop(sandbox_id, None)\n if info is None and sandbox_id in self._warm_pool:\n info, _ = self._warm_pool.pop(sandbox_id)\n else:\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n\n return sandbox, info, True\n\n def _drop_unhealthy_sandbox(self, sandbox_id: str, reason: str, *, expected_info: SandboxInfo | None = None) -> None:\n \"\"\"Remove and destroy a sandbox after a definitive failed health check.\"\"\"\n # Reserved for the whole path, not just the stop: this one untracks\n # first, so between the untrack and the `del:` claim an acquire misses\n # the caches and falls through to discovery, where `take()` still\n # succeeds against our own lease.\n if not self._reserve_local_teardown(sandbox_id, lambda: True):\n logger.info(f\"Skipped dropping sandbox {sandbox_id}: already being torn down by this instance\")\n return\n try:\n self._drop_unhealthy_reserved(sandbox_id, reason, expected_info=expected_info)\n finally:\n self._finish_local_teardown(sandbox_id)\n\n def _drop_unhealthy_reserved(self, sandbox_id: str, reason: str, *, expected_info: SandboxInfo | None = None) -> None:\n sandbox, info, removed = self._remove_tracked_sandbox(sandbox_id, expected_info=expected_info)\n if not removed:\n logger.info(f\"Skipped dropping sandbox {sandbox_id}: tracked info changed after health check\")\n return\n\n if sandbox is not None:\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing unhealthy sandbox {sandbox_id}: {e}\")\n\n if info is not None:\n # Gate this like every other reap path. The container failed a\n # definitive health check, but \"definitively dead to us\" is not proof\n # it is ours: a peer may have replaced the container behind this id,\n # in which case stopping it is the cross-instance kill again.\n if self._claim_ownership(sandbox_id, for_destroy=True):\n try:\n # Held like the other two stop paths: this one untracks before\n # claiming, so `_renew_owned_leases` cannot see the id either\n # and nothing else would refresh the marker. The heartbeat\n # releases the marker on exit (success or failure), so there is\n # no caller-side release to race a late refresh.\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(info)\n except Exception as e:\n logger.warning(f\"Error destroying unhealthy sandbox {sandbox_id}: {e}\")\n else:\n logger.info(\"Not destroying unhealthy sandbox %s: owned by another instance\", sandbox_id)\n\n logger.warning(f\"Dropped unhealthy sandbox {sandbox_id}: {reason}\")\n\n def _active_count_locked(self) -> int:\n \"\"\"Return active AIO sandbox count while ``_lock`` is held.\"\"\"\n return len(self._sandboxes)\n\n def _destroy_warm_entry(self, sandbox_id: str, entry: SandboxInfo, *, reason: str, still_reapable: Callable[[], bool]) -> bool:\n \"\"\"Destroy a warm-pool sandbox using AIO-specific backend logging.\n\n Claiming for destroy is the exclusion against **peers**: the lease is\n marked as a teardown, so a concurrent acquire on another instance is\n refused and the container cannot be re-acquired between this decision and\n the stop. That pairing is what replaced the per-sandbox flock guard. A\n claim that fails — peer-owned or backend unavailable — fails closed and\n we do not destroy.\n\n It is *not* an exclusion against this process: `claim()` succeeds against\n our own `own:` lease, so a same-process reclaim that ran before it wins\n the container and this stop lands on a turn already using it. The\n reservation is that half, and it is taken before the claim — after it,\n the entry stays visible in `_warm_pool` for the whole stop, so a reclaim\n would otherwise still find it.\n\n ``still_reapable`` is required rather than defaulting to unconditional:\n the safe default is the one that makes a new call site think about it,\n and this signature deliberately diverges from ``WarmPoolLifecycleMixin``'s\n hook for that reason. Safe because this provider overrides both mixin\n callers (``_evict_oldest_warm`` / ``_reap_expired_warm``); if those\n overrides were ever dropped, the mixin's call would fail loudly here\n rather than silently reopen the window.\n\n Returns:\n ``True`` when the container was stopped and the caller should drop\n its warm-pool entry; ``False`` when it is still running.\n \"\"\"\n if not self._reserve_local_teardown(sandbox_id, still_reapable):\n logger.info(\"Refusing to destroy warm-pool sandbox %s for %s: reclaimed by this instance\", sandbox_id, reason)\n return False\n\n try:\n if not self._claim_ownership(sandbox_id, for_destroy=True):\n logger.info(\"Refusing to destroy warm-pool sandbox %s for %s: owned by another instance\", sandbox_id, reason)\n return False\n\n try:\n # The marker must outlast the stop, not the TTL it was written with,\n # and is released by the heartbeat on exit. On a failed stop that\n # release matters just as much — the container is probably still up,\n # so a marker left behind would block its thread from re-acquiring it.\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(entry)\n except Exception as e:\n if reason == \"idle_timeout\":\n logger.error(f\"Failed to destroy idle warm-pool sandbox {sandbox_id}: {e}\")\n elif reason == \"replica_enforcement\":\n logger.error(f\"Failed to destroy warm-pool sandbox {sandbox_id}: {e}\")\n else:\n logger.error(f\"Failed to destroy warm-pool sandbox {sandbox_id} for {reason}: {e}\")\n return False\n\n # Remove the entry here, inside the reservation, rather than leaving\n # it to the caller. Releasing the reservation when the stop returns\n # and popping afterwards leaves a gap in which the container is\n # already stopped, the entry is still in `_warm_pool`, and nothing\n # marks it — so a reclaim picks it up and hands out a dead container.\n # The pop stays deferred relative to the *stop* (a refused or failed\n # stop keeps the entry), just no longer relative to the reservation.\n with self._lock:\n current = self._warm_pool.get(sandbox_id)\n if current is not None and current[0] is entry:\n self._warm_pool.pop(sandbox_id, None)\n self._warm_pool_identity.pop(sandbox_id, None)\n finally:\n self._finish_local_teardown(sandbox_id)\n\n if reason == \"idle_timeout\":\n logger.info(f\"Destroyed idle warm-pool sandbox {sandbox_id}\")\n elif reason == \"replica_enforcement\":\n logger.info(f\"Destroyed warm-pool sandbox {sandbox_id}\")\n else:\n logger.info(f\"Destroyed warm-pool sandbox {sandbox_id} for {reason}\")\n return True\n\n # ── Core: acquire / get / release / shutdown ─────────────────────────\n\n def acquire(self, thread_id: str | None = None, *, user_id: str | None = None) -> str:\n \"\"\"Acquire a sandbox environment and return its ID.\n\n For the same thread_id, this method will return the same sandbox_id\n across multiple turns, multiple processes, and (with shared storage)\n multiple pods.\n\n Thread-safe with both in-process and cross-process locking.\n\n Args:\n thread_id: Optional thread ID for thread-specific configurations.\n\n Returns:\n The ID of the acquired sandbox environment.\n \"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n if thread_id:\n thread_lock = self._get_thread_lock(thread_id, effective_user_id)\n with thread_lock:\n return self._acquire_internal(thread_id, user_id=effective_user_id)\n else:\n return self._acquire_internal(thread_id, user_id=effective_user_id)\n\n async def acquire_async(self, thread_id: str | None = None, *, user_id: str | None = None) -> str:\n \"\"\"Acquire a sandbox environment without blocking the event loop.\n\n Mirrors ``acquire()`` while keeping blocking backend operations off the\n event loop and using async-native readiness polling for newly created\n sandboxes.\n \"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n if thread_id:\n thread_lock = self._get_thread_lock(thread_id, effective_user_id)\n await _acquire_thread_lock_async(thread_lock)\n try:\n return await self._acquire_internal_async(thread_id, user_id=effective_user_id)\n finally:\n thread_lock.release()\n\n return await self._acquire_internal_async(thread_id, user_id=effective_user_id)\n\n def _acquire_internal(self, thread_id: str | None, *, user_id: str) -> str:\n \"\"\"Internal sandbox acquisition with two-layer consistency.\n\n Layer 1: In-process cache (fastest, covers same-process repeated access)\n Layer 2: Backend discovery (covers containers started by other processes;\n sandbox_id is deterministic from thread_id so no shared state file\n is needed — any process can derive the same container name)\n \"\"\"\n self._ensure_skills_projection(user_id)\n cached_id = self._reuse_in_process_sandbox(thread_id, user_id=user_id)\n if cached_id is not None:\n return cached_id\n\n # Deterministic ID for thread-specific, random for anonymous\n sandbox_id = self._sandbox_id_for_thread(thread_id, user_id)\n if thread_id:\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n self._assert_active_identity_available_locked(sandbox_id, key)\n\n # ── Layer 1.5: Warm pool (container still running, no cold-start) ──\n reclaimed_id = self._reclaim_warm_pool_sandbox(thread_id, sandbox_id, user_id=user_id)\n if reclaimed_id is not None:\n return reclaimed_id\n\n # ── Layer 2: Backend discovery + create (protected by cross-process lock) ──\n # Use a file lock so that two processes racing to create the same sandbox\n # for the same thread_id serialize here: the second process will discover\n # the container started by the first instead of hitting a name-conflict.\n if thread_id:\n return self._discover_or_create_with_lock(thread_id, sandbox_id, user_id=user_id)\n\n return self._create_sandbox(thread_id, sandbox_id, user_id=user_id)\n\n async def _acquire_internal_async(self, thread_id: str | None, *, user_id: str) -> str:\n \"\"\"Async counterpart to ``_acquire_internal``.\"\"\"\n await asyncio.to_thread(self._ensure_skills_projection, user_id)\n cached_id = await asyncio.to_thread(self._reuse_in_process_sandbox, thread_id, user_id=user_id)\n if cached_id is not None:\n return cached_id\n\n # Deterministic ID for thread-specific, random for anonymous\n sandbox_id = self._sandbox_id_for_thread(thread_id, user_id)\n if thread_id:\n key = self._thread_key(thread_id, user_id)\n with self._lock:\n self._assert_active_identity_available_locked(sandbox_id, key)\n\n # ── Layer 1.5: Warm pool (container still running, no cold-start) ──\n reclaimed_id = await asyncio.to_thread(self._reclaim_warm_pool_sandbox, thread_id, sandbox_id, user_id=user_id)\n if reclaimed_id is not None:\n return reclaimed_id\n\n # ── Layer 2: Backend discovery + create (protected by cross-process lock) ──\n if thread_id:\n return await self._discover_or_create_with_lock_async(thread_id, sandbox_id, user_id=user_id)\n\n return await self._create_sandbox_async(thread_id, sandbox_id, user_id=user_id)\n\n def _discover_or_create_with_lock(self, thread_id: str, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Discover an existing sandbox or create a new one under a cross-process file lock.\n\n The file lock serializes concurrent sandbox creation for the same thread_id\n across multiple processes, preventing container-name conflicts.\n \"\"\"\n paths = get_paths()\n effective_user_id = self._effective_acquire_user_id(user_id)\n paths.ensure_thread_dirs(thread_id, user_id=effective_user_id)\n lock_path = paths.thread_dir(thread_id, user_id=effective_user_id) / f\"{sandbox_id}.lock\"\n\n with open(lock_path, \"a\", encoding=\"utf-8\") as lock_file:\n locked = False\n try:\n _lock_file_exclusive(lock_file)\n locked = True\n # Re-check in-process caches under the file lock in case another\n # thread in this process won the race while we were waiting.\n cached_id = self._recheck_cached_sandbox(thread_id, sandbox_id, user_id=effective_user_id)\n if cached_id is not None:\n return cached_id\n\n # Backend discovery: another process may have created the container.\n discovered = self._backend.discover(sandbox_id)\n if discovered is not None:\n return self._register_discovered_sandbox(thread_id, discovered, user_id=effective_user_id)\n\n return self._create_sandbox(thread_id, sandbox_id, user_id=effective_user_id)\n finally:\n if locked:\n _unlock_file(lock_file)\n\n async def _discover_or_create_with_lock_async(self, thread_id: str, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Async counterpart to ``_discover_or_create_with_lock``.\"\"\"\n paths = get_paths()\n effective_user_id = self._effective_acquire_user_id(user_id)\n await asyncio.to_thread(paths.ensure_thread_dirs, thread_id, user_id=effective_user_id)\n lock_path = paths.thread_dir(thread_id, user_id=effective_user_id) / f\"{sandbox_id}.lock\"\n\n lock_file = await asyncio.to_thread(_open_lock_file, lock_path)\n locked = False\n try:\n await asyncio.to_thread(_lock_file_exclusive, lock_file)\n locked = True\n # Re-check in-process caches under the file lock in case another\n # thread in this process won the race while we were waiting.\n cached_id = await asyncio.to_thread(self._recheck_cached_sandbox, thread_id, sandbox_id, user_id=effective_user_id)\n if cached_id is not None:\n return cached_id\n\n # Backend discovery is sync because local discovery may inspect\n # Docker and perform a health check; keep it off the event loop.\n discovered = await asyncio.to_thread(self._backend.discover, sandbox_id)\n if discovered is not None:\n # Registration publishes ownership, which is blocking store IO\n # (filesystem or network depending on the backend) — same reason\n # every other step in this coroutine is offloaded.\n return await asyncio.to_thread(self._register_discovered_sandbox, thread_id, discovered, user_id=effective_user_id)\n\n return await self._create_sandbox_async(thread_id, sandbox_id, user_id=effective_user_id)\n finally:\n if locked:\n await asyncio.to_thread(_unlock_file, lock_file)\n await asyncio.to_thread(lock_file.close)\n\n def _destroy_unready_sandbox(self, sandbox_id: str, info: SandboxInfo) -> None:\n \"\"\"Tear down a freshly-created container whose readiness check failed.\n\n The container was started by the backend but never reached ready, so it\n never entered ``_register_created_sandbox`` and the ownership store has\n no lease for it yet. For the full readiness timeout (60s) it runs\n unowned, which is exactly the window a peer gateway's startup\n reconciliation is built to adopt across (#4206). Without a claim, a peer\n that adopts the not-yet-ready Pod and then has this instance's stop land\n on it is a cross-instance kill that interrupts an active turn (#4248).\n\n Claim the teardown lease first so this reap path is gated by the same\n ownership guard as every other destroy (``_destroy_warm_entry``,\n ``_drop_unhealthy_reserved``); fail closed (leave the container for the\n peer to reap via its own reconciliation) if a peer already owns it or\n the ownership store cannot answer.\n\n The claim alone is only the cross-**instance** half: it succeeds against\n our own lease by design, so it says nothing about this process. The\n same-process half is the local teardown reservation, taken first and\n held across the whole path — between the readiness timeout and the\n claim, ``_reconcile_orphans`` (idle checker, every 60s) can see this\n container running, untracked, and past its recovery grace, and park it\n in ``_warm_pool``; the subsequent claim would still succeed and the\n stop would land on an entry this instance has just adopted, leaving a\n dead warm entry for the next reclaim to hand out. The predicate checks\n the id is absent from both the active and warm maps; the reservation\n makes that check and the teardown mark one critical section, so no\n adopt/acquire can slip between them (same pairing as\n ``_destroy_warm_entry``).\n \"\"\"\n if not self._reserve_local_teardown(\n sandbox_id,\n lambda: sandbox_id not in self._sandboxes and sandbox_id not in self._sandbox_infos and sandbox_id not in self._warm_pool,\n ):\n logger.warning(\n \"Not destroying unready sandbox %s: adopted or being torn down by this instance\",\n sandbox_id,\n )\n return\n try:\n if not self._claim_ownership(sandbox_id, for_destroy=True):\n logger.warning(\n \"Not destroying unready sandbox %s: owned by another instance or ownership unavailable\",\n sandbox_id,\n )\n return\n try:\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(info)\n except Exception as e:\n logger.warning(f\"Error destroying unready sandbox {sandbox_id}: {e}\")\n finally:\n self._finish_local_teardown(sandbox_id)\n\n def _create_sandbox(self, thread_id: str | None, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Create a new sandbox via the backend.\n\n Args:\n thread_id: Optional thread ID.\n sandbox_id: The sandbox ID to use.\n\n Returns:\n The sandbox_id.\n\n Raises:\n RuntimeError: If sandbox creation or readiness check fails.\n \"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n extra_mounts = self._get_extra_mounts(thread_id, user_id=effective_user_id)\n provision_lark_cli_runtime = self._lark_integration_active(effective_user_id)\n provision_lark_cli_broker = self._lark_broker_active(effective_user_id)\n\n # Enforce replicas: only warm-pool containers count toward eviction budget.\n # Active sandboxes are in use by live threads and must not be forcibly stopped.\n replicas, total = self._replica_count()\n if total >= replicas:\n evicted = self._evict_oldest_warm()\n self._log_replicas_soft_cap(replicas, sandbox_id, evicted)\n\n info = self._backend.create(\n thread_id,\n sandbox_id,\n extra_mounts=extra_mounts or None,\n user_id=effective_user_id,\n provision_lark_cli_runtime=provision_lark_cli_runtime,\n provision_lark_cli_broker=provision_lark_cli_broker,\n )\n\n # Wait for sandbox to be ready\n if not wait_for_sandbox_ready(info.sandbox_url, timeout=60):\n # The container is running but unowned: ownership is published by\n # ``_register_created_sandbox`` after this gate. Claim the teardown\n # lease before stopping it so a peer cannot adopt the not-yet-ready\n # Pod in the meantime (#4248).\n self._destroy_unready_sandbox(sandbox_id, info)\n raise RuntimeError(f\"Sandbox {sandbox_id} failed to become ready within timeout at {info.sandbox_url}\")\n\n return self._register_created_sandbox(thread_id, sandbox_id, info, user_id=effective_user_id)\n\n async def _create_sandbox_async(self, thread_id: str | None, sandbox_id: str, *, user_id: str | None = None) -> str:\n \"\"\"Async counterpart to ``_create_sandbox``.\"\"\"\n effective_user_id = self._effective_acquire_user_id(user_id)\n extra_mounts = await asyncio.to_thread(self._get_extra_mounts, thread_id, user_id=effective_user_id)\n provision_lark_cli_runtime = await asyncio.to_thread(self._lark_integration_active, effective_user_id)\n provision_lark_cli_broker = await asyncio.to_thread(self._lark_broker_active, effective_user_id)\n\n # Enforce replicas: only warm-pool containers count toward eviction budget.\n # Active sandboxes are in use by live threads and must not be forcibly stopped.\n replicas, total = self._replica_count()\n if total >= replicas:\n evicted = await asyncio.to_thread(self._evict_oldest_warm)\n self._log_replicas_soft_cap(replicas, sandbox_id, evicted)\n\n info = await asyncio.to_thread(\n self._backend.create,\n thread_id,\n sandbox_id,\n extra_mounts=extra_mounts or None,\n user_id=effective_user_id,\n provision_lark_cli_runtime=provision_lark_cli_runtime,\n provision_lark_cli_broker=provision_lark_cli_broker,\n )\n\n # Wait for sandbox to be ready without blocking the event loop.\n if not await wait_for_sandbox_ready_async(info.sandbox_url, timeout=60):\n # The container is running but unowned: ownership is published by\n # ``_register_created_sandbox`` after this gate. Claim the teardown\n # lease before stopping it so a peer cannot adopt the not-yet-ready\n # Pod in the meantime (#4248).\n await asyncio.to_thread(self._destroy_unready_sandbox, sandbox_id, info)\n raise RuntimeError(f\"Sandbox {sandbox_id} failed to become ready within timeout at {info.sandbox_url}\")\n\n # Registration publishes ownership (blocking store IO), so it is offloaded\n # like every other blocking step on this path.\n return await asyncio.to_thread(self._register_created_sandbox, thread_id, sandbox_id, info, user_id=effective_user_id)\n\n def get(self, sandbox_id: str) -> Sandbox | None:\n \"\"\"Get a sandbox by ID. Updates last activity timestamp.\n\n Stays a pure in-memory lookup: async tool paths call this directly on the\n event loop (``ensure_sandbox_initialized_async``), so it must not touch\n the ownership store — that is blocking filesystem or network IO depending\n on the backend. Ownership is published off the event loop on\n acquire/reclaim and refreshed by the renewal thread (see\n ``_renew_owned_leases``).\n\n Args:\n sandbox_id: The ID of the sandbox.\n\n Returns:\n The sandbox instance if found, None otherwise.\n \"\"\"\n with self._lock:\n sandbox = self._sandboxes.get(sandbox_id)\n if sandbox is not None:\n self._last_activity[sandbox_id] = time.time()\n return sandbox\n\n def release(self, sandbox_id: str) -> None:\n \"\"\"Release a sandbox from active use into the warm pool.\n\n The container is kept running so it can be reclaimed quickly by the same\n thread on its next turn without a cold-start. The container will only be\n stopped when the replicas limit forces eviction or during shutdown.\n\n The host-side HTTP client owned by the cached ``AioSandbox`` instance is\n closed before the instance is dropped (#2872). The warm-pool entry only\n stores ``SandboxInfo``, so a fresh ``AioSandbox`` (and a fresh client)\n is constructed if the container is later reclaimed.\n\n Args:\n sandbox_id: The ID of the sandbox to release.\n \"\"\"\n info = None\n sandbox = None\n thread_keys_to_remove: list[tuple[str, str]] = []\n\n with self._lock:\n sandbox = self._sandboxes.pop(sandbox_id, None)\n info = self._sandbox_infos.pop(sandbox_id, None)\n thread_keys_to_remove = [key for key, sid in self._thread_sandboxes.items() if sid == sandbox_id]\n for key in thread_keys_to_remove:\n del self._thread_sandboxes[key]\n active_identity = self._active_sandbox_identity.pop(sandbox_id, None)\n self._last_activity.pop(sandbox_id, None)\n # Park in warm pool — container keeps running\n if info and sandbox_id not in self._warm_pool:\n self._warm_pool[sandbox_id] = (info, time.time())\n self._warm_pool_identity[sandbox_id] = thread_keys_to_remove[0] if thread_keys_to_remove else active_identity\n\n if sandbox is not None:\n # Defense-in-depth: close() already swallows its own errors; this\n # guard only protects against a future close() that misbehaves, so\n # host-side client cleanup can never block parking in the warm pool.\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} during release: {e}\")\n\n # Keep the lease while warm so a peer cannot adopt+destroy before we\n # reclaim, re-establishing it if it lapsed during a long turn. Never\n # raises: the turn is already over, so a store problem must not surface\n # through after_agent, and the renewal thread (which covers warm entries)\n # is the actual guarantee — this only narrows the window.\n if info is not None:\n # Same staleness as the renewal thread: the refresh is a store round\n # trip, and the thread's next turn can reclaim this warm entry while\n # it is in flight. Only drop it if nothing re-acquired it since.\n epoch = self._acquire_epoch_of(sandbox_id)\n if not self._refresh_ownership(sandbox_id):\n logger.warning(\"Sandbox %s is owned by another instance; releasing it from this warm pool\", sandbox_id)\n self._forget_lost_sandbox(sandbox_id, expected_epoch=epoch)\n\n logger.info(f\"Released sandbox {sandbox_id} to warm pool (container still running)\")\n\n def destroy(self, sandbox_id: str) -> None:\n \"\"\"Destroy a sandbox: stop the container and free all resources.\n\n Unlike release(), this actually stops the container. Use this for\n explicit cleanup, capacity-driven eviction, or shutdown.\n\n The host-side HTTP client owned by the cached ``AioSandbox`` instance is\n closed alongside backend/container destruction so no client/socket\n resources leak (#2872).\n\n Args:\n sandbox_id: The ID of the sandbox to destroy.\n \"\"\"\n self._destroy_tracked(sandbox_id, still_reapable=lambda: True)\n\n def _destroy_tracked(self, sandbox_id: str, *, still_reapable: Callable[[], bool]) -> None:\n \"\"\"``destroy()`` with a caller-supplied \"is this still reapable\" gate.\n\n Callers that decided to destroy *earlier* (the idle checker) pass their\n own predicate so the decision is re-validated in the same critical\n section that reserves the teardown. ``destroy()`` itself passes a\n constant: an explicit destroy is a decision made now.\n \"\"\"\n if not self._reserve_local_teardown(sandbox_id, still_reapable):\n logger.info(\"Skipping destroy of sandbox %s: re-acquired by this instance or already being torn down\", sandbox_id)\n return\n\n try:\n self._destroy_reserved(sandbox_id)\n finally:\n self._finish_local_teardown(sandbox_id)\n\n def _destroy_reserved(self, sandbox_id: str) -> None:\n # Claim before untracking. The reverse order loses the container on a\n # refused claim: still running, and no longer in any of our maps, so\n # nothing here would ever reap or reclaim it.\n if not self._claim_ownership(sandbox_id, for_destroy=True):\n logger.warning(\"Refusing to destroy sandbox %s: owned by another instance\", sandbox_id)\n return\n\n sandbox, info, _ = self._remove_tracked_sandbox(sandbox_id)\n\n if sandbox is not None:\n # Defense-in-depth: close() already swallows its own errors; this\n # guard only protects against a future close() that misbehaves, so\n # host-side client cleanup can never block container destruction.\n try:\n sandbox.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox {sandbox_id} during destroy: {e}\")\n\n if info:\n # The marker must outlast the stop, not the TTL it was written with,\n # and the heartbeat releases it on exit — on both outcomes. On a\n # failed stop the container is probably still up, so a marker left\n # behind would refuse its own thread's `take()` until the TTL lapses;\n # the error still propagates out of the `with` (`shutdown()` logs per\n # sandbox off it), it is just no longer this method's job to release.\n with self._held_teardown_lease(sandbox_id):\n self._backend.destroy(info)\n logger.info(f\"Destroyed sandbox {sandbox_id}\")\n else:\n # No container to stop, so no teardown lease was held: clear the\n # marker the claim above wrote, so an untracked id cannot leave a\n # lease stuck in `del:`.\n self._release_ownership(sandbox_id)\n\n def shutdown(self) -> None:\n \"\"\"Shutdown all sandboxes. Thread-safe and idempotent.\"\"\"\n with self._lock:\n if self._shutdown_called:\n return\n self._shutdown_called = True\n sandbox_ids = list(self._sandboxes.keys())\n warm_items = list(self._warm_pool.items())\n self._warm_pool.clear()\n self._warm_pool_identity.clear()\n\n self._stop_idle_checker()\n # Stop renewing before destroying: the destroy paths claim ownership\n # themselves, and a renewal racing them only re-publishes leases we are\n # about to drop.\n self._stop_lease_renewal()\n\n logger.info(f\"Shutting down {len(sandbox_ids)} active + {len(warm_items)} warm-pool sandbox(es)\")\n\n for sandbox_id in sandbox_ids:\n try:\n self.destroy(sandbox_id)\n except Exception as e:\n logger.error(f\"Failed to destroy sandbox {sandbox_id} during shutdown: {e}\")\n\n for sandbox_id, (info, _) in warm_items:\n # Route through _destroy_warm_entry so the ownership claim and the\n # container stop stay together, as on the idle path. Unconditional\n # here: the entries were removed from `_warm_pool` under the lock\n # above, so the pool-membership predicate the other callers use would\n # refuse every one of them.\n self._destroy_warm_entry(sandbox_id, info, reason=\"shutdown\", still_reapable=lambda: True)\n\n try:\n self._ownership.close()\n except Exception as e:\n logger.warning(f\"Error closing sandbox ownership store during shutdown: {e}\")\n"}}, "config.example.yaml": {"7025ccee403d016cfaeeb6f011475a370ff81e0d": {"t": 1785674627.810703, "n": 119502, "h": "5cb81772d930d279", "c": "# Configuration for the DeerFlow application\n#\n# Guidelines:\n# - Copy this file to `config.yaml` and customize it for your environment\n# - The default path of this configuration file is `config.yaml` in the project root.\n# You can set `DEER_FLOW_PROJECT_ROOT` to define that root explicitly, or use\n# `DEER_FLOW_CONFIG_PATH` to point at a specific config file.\n# - Runtime state defaults to `.deer-flow` under the project root. Override it\n# with `DEER_FLOW_HOME` when you need a different writable data directory.\n# - Environment variables are available for all field values. Example: `api_key: $OPENAI_API_KEY`\n# - The `use` path is a string that looks like \"package_name.sub_package_name.module_name:class_name/variable_name\".\n\n# ============================================================================\n# Config Version (used to detect outdated config files)\n# ============================================================================\n# Bump this number when the config schema changes.\n# Run `make config-upgrade` to merge new fields into your local config.yaml.\nconfig_version: 31\n\n# ============================================================================\n# Logging\n# ============================================================================\n# Log level for deerflow modules (debug/info/warning/error)\nlog_level: info\n\n# Request trace correlation for Gateway logs, HTTP response headers, and\n# Langfuse metadata. Disabled by default to preserve existing HTTP/log output.\nlogging:\n enhance:\n enabled: false\n format: text\n\n# ============================================================================\n# Agent Extensions\n# ============================================================================\n# Optional AgentMiddleware classes loaded into the lead and subagent runtime\n# middleware chains after built-in runtime middlewares, but before the\n# safety/clarification tail. Missing packages, invalid classes, and broken\n# modules fail loudly at agent creation with an actionable import error.\n# The same zero-argument class list applies to both lead and subagent runtimes;\n# lead-only vs subagent-only configuration is not expressible yet. Treat these\n# files as trusted operator config because middleware classes execute code.\n# Uncomment this block to define middlewares in config.yaml. Leaving it commented\n# lets extensions_config.json remain the source of truth for extension packages.\n# extensions:\n# middlewares:\n# - my_company.deerflow_middlewares:DomainGuardMiddleware\n# - my_company.deerflow_middlewares:LatencyStampingMiddleware\n\n# ============================================================================\n# Tracing / Observability (Monocle)\n# ============================================================================\n# Optional agent tracing via Monocle. Configured through environment variables\n# (MONOCLE_TRACING, MONOCLE_EXPORTERS, OKAHU_API_KEY — like LangSmith/Langfuse),\n# not config.yaml keys, and OFF by default. See README.md → \"Monocle Tracing\"\n# for setup, what each exporter captures, and where the trace data goes.\n\n# ============================================================================\n# Token Usage\n# ============================================================================\n# Enable token usage collection and display.\n# When enabled, DeerFlow records input/output/total tokens per model call\n# and shows usage metadata in the workspace UI when providers return it.\ntoken_usage:\n enabled: true\n\n# ============================================================================\n# Token Budget — Per-run token limits\n# ============================================================================\n# Prevents runaway API costs by enforcing hard token limits per run.\n# When warn_threshold is crossed, the agent receives an in-context warning.\n# When hard_stop_threshold is crossed, tool_calls are stripped and the agent\n# is forced to produce a final answer immediately.\ntoken_budget:\n enabled: false # Set to true to activate budget enforcement\n max_tokens: 200000 # Total token limit (input + output) per run\n max_input_tokens: null # Optional separate input-only limit\n max_output_tokens: null # Optional separate output-only limit\n warn_threshold: 0.8 # Warn at 80% of the budget\n hard_stop_threshold: 1.0 # Force stop at 100% of the budget\n\n# ============================================================================\n# Recursion Limit — Hard ceiling for a client-supplied run recursion_limit\n# ============================================================================\n# A run's recursion_limit caps the number of LangGraph super-steps (each is at\n# least one LLM call). The Gateway never trusts a client-supplied value\n# verbatim: any value above this ceiling is clamped down to it, preventing\n# runaway API cost / DoS. Invalid or non-positive client values fall back to\n# the server default of 100. Raise this only if you legitimately run very\n# deeply nested subagent graphs.\nmax_recursion_limit: 1000\n\n\n# ============================================================================\n# Models Configuration\n# ============================================================================\n# Configure available LLM models for the agent to use.\n#\n# Two token fields look similar but mean different things:\n# - `max_tokens` is the per-call OUTPUT cap passed to the provider.\n# - `context_window` is a positive integer for the total context capacity\n# (prompt + completion) and drives the real-time \"% context used\" indicator\n# in the chat UI.\n# Leave `context_window` unset if the provider limit is unknown; the percentage\n# will not render. Verify configured values against the provider's model docs.\n#\n# Optional per-model pricing (powers the real-cost display on the workspace\n# console). Add a `pricing` block to any model entry; use ONE currency across\n# all models. Mixed currencies disable cost reporting to prevent invalid sums.\n# Prices are per one million tokens.\n#\n# pricing:\n# currency: CNY # ISO code shown in the console (CNY, USD, ...)\n# input_per_million: 8.0 # price per 1M input tokens (cache miss)\n# output_per_million: 32.0 # price per 1M output tokens\n# input_cache_hit_per_million: 0.8 # price per 1M cache-hit input tokens\n# # (optional; omit → hits billed at miss price)\n\nmodels:\n # Example: Volcengine (Doubao) model\n # - name: doubao-seed-1.8\n # display_name: Doubao-Seed-1.8\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: doubao-seed-1-8-251228\n # api_base: https://ark.cn-beijing.volces.com/api/v3\n # api_key: $VOLCENGINE_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # context_window: 262144 # Total prompt + completion capacity\n # supports_thinking: true\n # supports_vision: true\n # supports_reasoning_effort: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Volcengine Coding Plan (one key, multi-vendor gateway)\n # The Coding Plan endpoint (/api/coding/v3) lets you access models from\n # Doubao, GLM, DeepSeek, Kimi, and MiniMax with a single API key.\n # Each model may differ in thinking/vision support - configure per-model.\n #\n # - name: glm-5.2-cp\n # display_name: GLM-5.2 (Coding Plan)\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: glm-5.2\n # api_base: https://ark.cn-beijing.volces.com/api/coding/v3\n # api_key: $VOLCENGINE_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # supports_thinking: true\n # supports_vision: false\n # supports_reasoning_effort: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n #\n # - name: deepseek-v4-pro-cp\n # display_name: DeepSeek-V4-Pro (Coding Plan)\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: deepseek-v4-pro\n # api_base: https://ark.cn-beijing.volces.com/api/coding/v3\n # api_key: $VOLCENGINE_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # supports_thinking: true\n # supports_vision: false\n # supports_reasoning_effort: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n # Example: OpenAI model\n # - name: gpt-4\n # display_name: GPT-4\n # use: langchain_openai:ChatOpenAI\n # model: gpt-4\n # api_key: $OPENAI_API_KEY # Use environment variable\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096 # Per-call output cap\n # context_window: 128000 # Total prompt + completion capacity\n # temperature: 0.7\n # supports_vision: true # Enable vision support for view_image tool\n\n # Example: OpenAI Responses API model\n # - name: gpt-5-responses\n # display_name: GPT-5 (Responses API)\n # use: langchain_openai:ChatOpenAI\n # model: gpt-5\n # api_key: $OPENAI_API_KEY\n # request_timeout: 600.0\n # max_retries: 2\n # use_responses_api: true\n # output_version: responses/v1\n # context_window: 400000\n # supports_vision: true\n\n # Example: Ollama (native provider — preserves thinking/reasoning content)\n #\n # IMPORTANT: Use langchain_ollama:ChatOllama instead of langchain_openai:ChatOpenAI\n # for Ollama models. The OpenAI-compatible endpoint (/v1/chat/completions) does NOT\n # return reasoning_content as a separate field — thinking content is either flattened\n # into tags or dropped entirely (ollama/ollama#15293). The native Ollama API\n # (/api/chat) correctly separates thinking from response content.\n #\n # Install: cd backend && uv pip install 'deerflow-harness[ollama]'\n #\n # - name: qwen3-local\n # display_name: Qwen3 32B (Ollama)\n # use: langchain_ollama:ChatOllama\n # model: qwen3:32b\n # base_url: http://localhost:11434 # No /v1 suffix — uses native /api/chat\n # num_predict: 8192\n # temperature: 0.7\n # reasoning: true # Passes think:true to Ollama native API\n # context_window: 32768 # Match the context length served by Ollama\n # supports_thinking: true\n # supports_vision: false\n #\n # - name: gemma4-local\n # display_name: Gemma 4 27B (Ollama)\n # use: langchain_ollama:ChatOllama\n # model: gemma4:27b\n # base_url: http://localhost:11434\n # num_predict: 8192\n # temperature: 0.7\n # reasoning: true\n # context_window: 32768\n # supports_thinking: true\n # supports_vision: true\n #\n # For Docker deployments, use host.docker.internal instead of localhost:\n # base_url: http://host.docker.internal:11434\n\n # Example: Anthropic Claude model (with extended thinking)\n # supports_thinking: true is required — without it, DeerFlow silently falls\n # back to non-thinking mode even when the UI thinking toggle is on.\n # budget_tokens is required by the Anthropic API when thinking.type=enabled\n # (no server default; min 1024; must be less than max_tokens).\n # - name: claude-sonnet-4\n # display_name: Claude Sonnet 4\n # use: langchain_anthropic:ChatAnthropic\n # model: claude-sonnet-4-20250514\n # api_key: $ANTHROPIC_API_KEY\n # default_request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 16000 # Per-call output cap\n # context_window: 200000 # Total prompt + completion capacity\n # supports_vision: true\n # supports_thinking: true\n # when_thinking_enabled:\n # thinking:\n # type: enabled\n # budget_tokens: 4096 # required; min 1024; must be < max_tokens\n # when_thinking_disabled:\n # thinking:\n # type: disabled\n\n # Example: Google Gemini model (native SDK, no thinking support)\n # - name: gemini-2.5-pro\n # display_name: Gemini 2.5 Pro\n # use: langchain_google_genai:ChatGoogleGenerativeAI\n # model: gemini-2.5-pro\n # gemini_api_key: $GEMINI_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # context_window: 1048576\n # supports_vision: true\n\n # Example: Gemini model via OpenAI-compatible gateway (with thinking support)\n # Use PatchedChatOpenAI so that tool-call thought_signature values on tool_calls\n # are preserved across multi-turn tool-call conversations — required by the\n # Gemini API when thinking is enabled. See:\n # https://docs.cloud.google.com/vertex-ai/generative-ai/docs/thought-signatures\n # - name: gemini-2.5-pro-thinking\n # display_name: Gemini 2.5 Pro (Thinking)\n # use: deerflow.models.patched_openai:PatchedChatOpenAI\n # model: google/gemini-2.5-pro-preview # model name as expected by your gateway\n # api_key: $GEMINI_API_KEY\n # base_url: https:///v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 16384\n # context_window: 1048576\n # supports_thinking: true\n # supports_vision: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Xiaomi MiMo model (with thinking support)\n # MiMo thinking mode returns reasoning_content and requires that field to be\n # replayed on historical assistant messages in multi-turn agent/tool-call\n # conversations. Use PatchedChatMiMo instead of plain ChatOpenAI.\n # Use https://api.xiaomimimo.com/v1 with pay-as-you-go `sk-...` keys.\n # Use your Token Plan regional URL (for example\n # https://token-plan-cn.xiaomimimo.com/v1) with Token Plan `tp-...` keys.\n # PatchedChatMiMo is model-id agnostic; use it for every MiMo thinking model\n # entry you configure (for example mimo-v2.5-pro, mimo-v2.5, mimo-v2-pro,\n # mimo-v2-omni, or mimo-v2-flash), including models referenced by subagent\n # model overrides.\n # See: https://platform.xiaomimimo.com/docs/en-US/usage-guide/passing-back-reasoning_content\n # - name: mimo-v2.5-pro\n # display_name: MiMo V2.5 Pro\n # use: deerflow.models.patched_mimo:PatchedChatMiMo\n # model: mimo-v2.5-pro\n # api_key: $MIMO_API_KEY\n # base_url: https://api.xiaomimimo.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: DeepSeek V4 model (with thinking support)\n # - name: deepseek-v4\n # display_name: DeepSeek V4 (Thinking)\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: deepseek-v4-pro\n # api_key: $DEEPSEEK_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false # DeepSeek V4 does not support vision\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Kimi K2.5 model\n # - name: kimi-k2.5\n # display_name: Kimi K2.5\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: kimi-k2.5\n # api_base: https://api.moonshot.cn/v1\n # api_key: $MOONSHOT_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # max_tokens: 32768\n # supports_thinking: true\n # supports_vision: true # Check your specific model's capabilities\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Novita AI (OpenAI-compatible)\n # Novita provides an OpenAI-compatible API with competitive pricing\n # See: https://novita.ai\n # - name: novita-deepseek-v3.2\n # display_name: Novita DeepSeek V3.2\n # use: langchain_openai:ChatOpenAI\n # model: deepseek/deepseek-v3.2\n # api_key: $NOVITA_API_KEY\n # base_url: https://api.novita.ai/openai\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 0.7\n # supports_thinking: true\n # supports_vision: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: StepFun (阶跃星辰) reasoning models\n # StepFun provides OpenAI-compatible API with reasoning models.\n # With reasoning_format: deepseek-style, the API returns reasoning_content\n # (same field as DeepSeek), which must be replayed on historical assistant\n # messages in multi-turn tool-call conversations.\n # Use PatchedChatStepFun instead of plain ChatOpenAI.\n # Docs: https://platform.stepfun.com/docs/api-reference/chat-completions\n # - name: step-3.7-flash\n # display_name: Step 3.7 Flash\n # use: deerflow.models.patched_stepfun:PatchedChatStepFun\n # model: step-3.7-flash\n # api_key: $STEPFUN_API_KEY\n # base_url: https://api.stepfun.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # supports_thinking: true\n # supports_reasoning_effort: true\n # supports_vision: true\n # when_thinking_enabled:\n # extra_body:\n # reasoning_format: deepseek-style\n # when_thinking_disabled:\n # extra_body:\n # reasoning_format: deepseek-style\n\n # Example: MiniMax (OpenAI-compatible) - International Edition\n # MiniMax provides high-performance models with 512K context window and 128K max output\n # Docs: https://platform.minimax.io/docs/api-reference/text-openai-api\n # - name: minimax-m3\n # display_name: MiniMax M3\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M3\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimax.io/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: true\n # supports_thinking: true\n # # PatchedChatMiniMax is the MiniMax adapter: it enables reasoning_split and\n # # maps MiniMax's structured reasoning into reasoning_content (the field\n # # DeerFlow understands), and it strips the per-message `name` field that\n # # DeerFlow middlewares attach — MiniMax rejects requests whose user-message\n # # names differ with \"user name must be consistent (2013)\". Declare the\n # # thinking toggle so non-thinking paths (flash mode, follow-up suggestions,\n # # title/memory generation) truly disable reasoning instead of spending\n # # tokens on it.\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: adaptive\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # NOTE: M2.x models always think — passing thinking:{type:disabled} has no\n # effect (per MiniMax docs), so the toggle above is omitted for M2.7. The\n # follow-up-suggestions endpoint strips inline defensively regardless.\n # Still use the PatchedChatMiniMax adapter: it strips the per-message `name`\n # field DeerFlow middlewares attach, which MiniMax otherwise rejects with\n # \"user name must be consistent (2013)\".\n # - name: minimax-m2.7\n # display_name: MiniMax M2.7\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimax.io/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # - name: minimax-m2.7-highspeed\n # display_name: MiniMax M2.7 Highspeed\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7-highspeed\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimax.io/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # Example: MiniMax (OpenAI-compatible) - CN 中国区用户\n # MiniMax provides high-performance models with 512K context window and 128K max output\n # Docs: https://platform.minimaxi.com/docs/api-reference/text-openai-api\n # - name: minimax-m3\n # display_name: MiniMax M3\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M3\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimaxi.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: true\n # supports_thinking: true\n # # PatchedChatMiniMax is the MiniMax adapter: it enables reasoning_split and\n # # maps MiniMax's structured reasoning into reasoning_content (the field\n # # DeerFlow understands), and it strips the per-message `name` field that\n # # DeerFlow middlewares attach — MiniMax rejects requests whose user-message\n # # names differ with \"user name must be consistent (2013)\". Declare the\n # # thinking toggle so non-thinking paths (flash mode, follow-up suggestions,\n # # title/memory generation) truly disable reasoning instead of spending\n # # tokens on it.\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: adaptive\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # NOTE: M2.x models always think — passing thinking:{type:disabled} has no\n # effect (per MiniMax docs), so the toggle above is omitted for M2.7. The\n # follow-up-suggestions endpoint strips inline defensively regardless.\n # Still use the PatchedChatMiniMax adapter: it strips the per-message `name`\n # field DeerFlow middlewares attach, which MiniMax otherwise rejects with\n # \"user name must be consistent (2013)\".\n # - name: minimax-m2.7\n # display_name: MiniMax M2.7\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimaxi.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # - name: minimax-m2.7-highspeed\n # display_name: MiniMax M2.7 Highspeed\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7-highspeed\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimaxi.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # Example: OpenRouter (OpenAI-compatible)\n # OpenRouter models use the same ChatOpenAI + base_url pattern as other OpenAI-compatible gateways.\n # - name: openrouter-gemini-2.5-flash\n # display_name: Gemini 2.5 Flash (OpenRouter)\n # use: langchain_openai:ChatOpenAI\n # model: google/gemini-2.5-flash-preview\n # api_key: $OPENAI_API_KEY\n # base_url: https://openrouter.ai/api/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # temperature: 0.7\n\n # Example: Atlas Cloud (OpenAI-compatible)\n # Atlas Cloud exposes a single OpenAI-compatible endpoint in front of many open\n # models (DeepSeek, Qwen, Kimi, GLM, MiniMax, Llama, ...), so it uses the same\n # ChatOpenAI + base_url pattern as other OpenAI-compatible gateways.\n # Browse model ids at https://api.atlascloud.ai/v1/models — see https://atlascloud.ai\n # - name: atlascloud-deepseek-v3.2\n # display_name: DeepSeek V3.2 (Atlas Cloud)\n # use: langchain_openai:ChatOpenAI\n # model: deepseek-ai/DeepSeek-V3.2-Exp\n # api_key: $ATLASCLOUD_API_KEY\n # base_url: https://api.atlascloud.ai/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # temperature: 0.7\n # supports_vision: false\n #\n # For reasoning models on Atlas Cloud (e.g. a Qwen3 *-thinking id), use the\n # patched OpenAI-compatible adapter so reasoning_content is replayed across\n # multi-turn tool-call conversations:\n # - name: atlascloud-qwen3-thinking\n # display_name: Qwen3 235B Thinking (Atlas Cloud)\n # use: deerflow.models.patched_openai:PatchedChatOpenAI\n # model: qwen/qwen3-235b-a22b-thinking-2507\n # api_key: $ATLASCLOUD_API_KEY\n # base_url: https://api.atlascloud.ai/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: vLLM 0.19.0 (OpenAI-compatible, with reasoning toggle)\n # DeerFlow's vLLM provider preserves vLLM reasoning across tool-call turns and\n # toggles Qwen-style reasoning by writing\n # extra_body.chat_template_kwargs.enable_thinking=true/false.\n # Some reasoning models also require the server to be started with\n # `vllm serve ... --reasoning-parser `.\n # - name: qwen3-32b-vllm\n # display_name: Qwen3 32B (vLLM)\n # use: deerflow.models.vllm_provider:VllmChatModel\n # model: Qwen/Qwen3-32B\n # api_key: $VLLM_API_KEY\n # base_url: http://localhost:8000/v1\n # # Enable only when the endpoint reports cumulative usage on every stream chunk.\n # cumulative_stream_usage: true\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false\n # when_thinking_enabled:\n # extra_body:\n # chat_template_kwargs:\n # enable_thinking: true\n\n\n # Example: Qwen3-Coder deployed on MindIE Engine\n # - name: Qwen3_Coder_480B_MindIE\n # display_name: Qwen3-Coder-480B (MindIE)\n # use: deerflow.models.mindie_provider:MindIEChatModel\n # model: Qwen3-Coder-480B-A35B-Instruct-Client\n # base_url: http://localhost:8989/v1\n # api_key: $OPENAI_API_KEY\n # temperature: 0\n # max_retries: 1\n # supports_thinking: false\n # supports_vision: false\n # supports_reasoning_effort: false\n # # --- Advanced Network Settings ---\n # # Due to MindIE's streaming limitations with tool calling, the provider\n # # uses mock-streaming (awaiting full generation). Extended timeouts are required.\n # read_timeout: 900.0 # 15 minutes to prevent drops during long document generation\n # connect_timeout: 30.0\n # write_timeout: 60.0\n # pool_timeout: 30.0\n\n# ============================================================================\n# Tool Groups Configuration\n# ============================================================================\n# Define groups of tools for organization and access control\n\ntool_groups:\n - name: web\n - name: file:read\n - name: file:write\n - name: bash\n - name: browser\n\n# ============================================================================\n# Tools Configuration\n# ============================================================================\n# Configure available tools for the agent to use\n\ntools:\n # Web search tool (uses DuckDuckGo, no API key required)\n - name: web_search\n group: web\n use: deerflow.community.ddg_search.tools:web_search_tool\n max_results: 5\n # backend: auto # DDGS backend(s): auto, duckduckgo, brave, wikipedia, etc.\n # region: wt-wt # wt-wt is normalized for Wikipedia when backend includes auto/all/wikipedia.\n # safesearch: moderate # on, moderate, off\n\n # Web search tool (uses SearXNG, self-hosted, no API key required)\n # SearXNG is a free internet metasearch engine which aggregates results from\n # various search services. Deploy your own instance: https://github.com/searxng/searxng\n # For Docker deployments, use the Docker service name instead of localhost.\n # - name: web_search\n # group: web\n # use: deerflow.community.searxng.tools:web_search_tool\n # base_url: http://localhost:8088 # SearXNG instance URL (default: :8088; Docker: http://searxng:8080)\n # max_results: 5 # Maximum number of search results\n\n # Web search tool (uses Serper - Google Search API, requires SERPER_API_KEY)\n # Serper provides real-time Google Search results. Sign up at https://serper.dev\n # Note: set SERPER_API_KEY in your environment before starting the app.\n # Avoid putting literal API keys in config.yaml; use the $VAR form instead.\n # - name: web_search\n # group: web\n # use: deerflow.community.serper.tools:web_search_tool\n # max_results: 5 # capped at 10 by the Serper provider\n # # api_key: $SERPER_API_KEY # Optional explicit env-var reference\n\n # Web search tool (uses Brave Search API, requires BRAVE_SEARCH_API_KEY)\n # Brave Search returns results from an independent index. Sign up at\n # https://brave.com/search/api/ to get a key. Unlike the DuckDuckGo\n # `backend: brave` option above, this calls the official Brave API directly.\n # - name: web_search\n # group: web\n # use: deerflow.community.brave.tools:web_search_tool\n # max_results: 5 # Capped at 20 by the Brave Search API\n # # api_key: $BRAVE_SEARCH_API_KEY # Optional if the env var is set\n\n # Web search tool (requires Tavily API key)\n # - name: web_search\n # group: web\n # use: deerflow.community.tavily.tools:web_search_tool\n # max_results: 5\n # # api_key: $TAVILY_API_KEY # Set if needed\n\n # Web search tool (uses InfoQuest, requires InfoQuest API key)\n # - name: web_search\n # group: web\n # use: deerflow.community.infoquest.tools:web_search_tool\n # # Used to limit the scope of search results, only returns content within the specified time range. Set to -1 to disable time filtering\n # search_time_range: 10\n\n # Web search tool (uses Exa, requires EXA_API_KEY)\n # - name: web_search\n # group: web\n # use: deerflow.community.exa.tools:web_search_tool\n # max_results: 5\n # search_type: auto # Options: auto, neural, keyword\n # contents_max_characters: 1000\n # # api_key: $EXA_API_KEY\n\n # Web search tool (uses Firecrawl, requires FIRECRAWL_API_KEY)\n # - name: web_search\n # group: web\n # use: deerflow.community.firecrawl.tools:web_search_tool\n # max_results: 5\n # # api_key: $FIRECRAWL_API_KEY\n\n # Web search tool (uses GroundRoute, requires GROUNDROUTE_API_KEY)\n # GroundRoute is a meta search layer: one API in front of six engines (Serper,\n # Brave, Exa, Tavily, Firecrawl, Perplexity). It routes each query to the cheapest\n # engine that clears a quality bar and fails over if one is down. Pricing is\n # gain-share (you keep about half of any cache savings). Get a key at\n # https://groundroute.ai/keys\n # - name: web_search\n # group: web\n # use: deerflow.community.groundroute.tools:web_search_tool\n # max_results: 5 # Clamped to 1-50 by GroundRoute\n # # api_key: $GROUNDROUTE_API_KEY # Optional if the env var is set\n\n # Web search tool (uses fastCRW - Firecrawl-compatible web scraper, single binary,\n # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.)\n # - name: web_search\n # group: web\n # use: deerflow.community.fastcrw.tools:web_search_tool\n # max_results: 5\n # # api_key: $CRW_API_KEY\n # # base_url: https://fastcrw.com/api # default cloud; set to e.g. http://localhost:3000 for self-host\n\n # Web fetch tool (uses Browserless - headless Chrome, self-hosted or cloud)\n # Browserless renders pages with a real headless Chrome, ideal for JavaScript-heavy\n # sites and SPAs. Deploy your own: https://github.com/browserless/browserless\n # For Docker deployments, use the Docker service name instead of localhost.\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry below before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.browserless.tools:web_fetch_tool\n # base_url: http://localhost:3032 # Browserless instance URL (default: :3032; Docker: http://browserless:3000)\n # # token: $BROWSERLESS_TOKEN # API token (required for Browserless Cloud; optional for self-hosted)\n # timeout_s: 30 # Request timeout in seconds\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets.\n # # wait_for_event: \"networkidle\" # Wait for a page event before returning (e.g. \"load\", \"networkidle\")\n # # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds\n # # wait_for_selector: \"article\" # CSS selector to wait for before returning\n\n # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no third-party API key)\n # Crawl4AI returns server-cleaned \"fit\" markdown directly (no readability step needed),\n # ideal for JavaScript-heavy sites. Self-host it:\n # docker run -d -p 11235:11235 --shm-size=1g \\\n # -e CRAWL4AI_API_TOKEN=$CRAWL4AI_TOKEN unclecode/crawl4ai:0.9.2\n # Crawl4AI >= 0.9 is secure-by-default: a bearer token is REQUIRED on every request\n # except GET /health, and a server started without CRAWL4AI_API_TOKEN binds 127.0.0.1\n # only (so a container/remote DeerFlow cannot reach it at all). Set the same value in\n # the server env and in `token:` below, or requests fail with HTTP 401.\n # Use >= 0.8.7: earlier images, including 0.8.6, carry known pre-auth RCEs.\n # For Docker deployments, use the Docker service name instead of localhost.\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry below before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.crawl4ai.tools:web_fetch_tool\n # base_url: http://localhost:11235 # Crawl4AI server URL (Docker: http://crawl4ai:11235)\n # token: $CRAWL4AI_TOKEN # Bearer token; required by Crawl4AI >= 0.9\n # timeout: 30 # Request timeout in seconds\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets.\n # # filter: fit # Markdown filter: fit (default) | raw | bm25 | llm\n\n # Web capture tool (uses Browserless /screenshot to render a page as an artifact)\n # Browserless captures JavaScript-heavy pages with a real headless Chrome and\n # writes the screenshot into the current thread's outputs. It can run against\n # a self-hosted Browserless instance without a token, or Browserless Cloud with\n # BROWSERLESS_TOKEN. For Docker deployments, use the Docker service name instead\n # of localhost.\n # - name: web_capture\n # group: web\n # use: deerflow.community.browserless.tools:web_capture_tool\n # base_url: http://localhost:3032 # Browserless instance URL (Docker: http://browserless:3000)\n # # token: $BROWSERLESS_TOKEN # Required for Browserless Cloud; optional for self-hosted\n # timeout_s: 30 # Request timeout in seconds (`timeout` also accepted)\n # output_format: png # png, jpeg, or webp\n # full_page: true # Capture entire page instead of viewport only\n # viewport_width: 1280\n # viewport_height: 720\n # # wait_for_selector: \"main\" # CSS selector to wait for before capturing\n # # wait_for_selector_timeout_ms: 5000\n # # wait_for_timeout_ms: 1000 # Extra wait after navigation in milliseconds\n # # best_attempt: true # Continue with current page state if waits time out\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY to\n # # # capture internal/private targets (loopback, RFC1918, etc.)\n\n # Agentic browser control (stateful navigate → observe → click/type loop).\n # Unlike web_fetch/web_capture (read-only, stateless), these tools keep a live\n # per-thread Playwright browser so the agent can interact with JavaScript-heavy\n # pages, fill forms, and follow multi-step flows. Elements are addressed by the\n # numeric [ref] index returned in each page snapshot.\n #\n # Requires the optional dependency, installed once per environment:\n # cd backend && uv sync --extra browser && uv run playwright install chromium\n # make dev / Docker startup auto-detect an active browser_navigate entry and\n # preserve the browser extra during dependency sync.\n #\n # Uncomment the whole block to enable. Launch and URL-policy settings are read\n # canonically from browser_navigate and shared by every browser tool + Live.\n # - name: browser_navigate\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_navigate_tool\n # headless: true # Set false only for local, trusted debugging\n # timeout_ms: 30000 # Per-action navigation/interaction timeout\n # viewport_width: 1280\n # viewport_height: 720\n # # allow_private_addresses: false # SSRF guard: keep false in production\n # # cdp_url: http://127.0.0.1:9222 # Attach to YOUR running Chrome (Codex-style)\n # # allow_unguarded_cdp: false # REQUIRED true with cdp_url: attaching to an\n # # # existing Chrome prevents DeerFlow from\n # # # enforcing its subresource/redirect SSRF guard.\n # # # instead of launching a private headless one.\n # # # Start Chrome with --remote-debugging-port=9222\n # # # so you watch the agent drive your real browser\n # # # with your real login sessions. Local/trusted only;\n # # # never enable for an untrusted CDP endpoint.\n # # # Chrome must be version-matched to the bundled\n # # # Playwright; a much newer Chrome can reject the\n # # # CDP handshake. If so, use headless: false to launch\n # # # Playwright's own visible browser instead.\n # # Multi-worker note: browser sessions live in one Gateway worker's memory.\n # # Keep GATEWAY_WORKERS=1 while this tool group is enabled.\n # - name: browser_snapshot\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_snapshot_tool\n # - name: browser_click\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_click_tool\n # - name: browser_type\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_type_tool\n # - name: browser_get_text\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_get_text_tool\n # max_chars: 8000 # Truncation cap for browser_get_text output\n # - name: browser_back\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_back_tool\n # - name: browser_screenshot\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_screenshot_tool\n # - name: browser_close\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_close_tool\n\n # Web fetch tool (uses Exa)\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry below before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.exa.tools:web_fetch_tool\n # # api_key: $EXA_API_KEY\n\n # Web fetch tool (uses Jina AI reader)\n - name: web_fetch\n group: web\n use: deerflow.community.jina_ai.tools:web_fetch_tool\n timeout: 10\n # Optional proxy for restricted networks / Docker / WSL.\n # Use host.docker.internal instead of 127.0.0.1 when the proxy runs on the host.\n # proxy: $HTTPS_PROXY\n # trust_env: true\n\n # Web fetch tool (uses InfoQuest)\n # - name: web_fetch\n # group: web\n # use: deerflow.community.infoquest.tools:web_fetch_tool\n # # Overall timeout for the entire crawling process (in seconds). Set to positive value to enable, -1 to disable\n # timeout: 10\n # # Waiting time after page loading (in seconds). Set to positive value to enable, -1 to disable\n # fetch_time: 10\n # # Timeout for navigating to the page (in seconds). Set to positive value to enable, -1 to disable\n # navigation_timeout: 30\n\n # Web fetch tool (uses Firecrawl, requires FIRECRAWL_API_KEY)\n # - name: web_fetch\n # group: web\n # use: deerflow.community.firecrawl.tools:web_fetch_tool\n # # api_key: $FIRECRAWL_API_KEY\n\n # Web fetch tool (uses GroundRoute, requires GROUNDROUTE_API_KEY)\n # Fetches a page's extracted text via GroundRoute mode=page.\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry above before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.groundroute.tools:web_fetch_tool\n # # api_key: $GROUNDROUTE_API_KEY\n\n # Web fetch tool (uses fastCRW - Firecrawl-compatible web scraper, single binary,\n # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.)\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry above before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.fastcrw.tools:web_fetch_tool\n # # api_key: $CRW_API_KEY\n # # base_url: https://fastcrw.com/api # default cloud; set to e.g. http://localhost:3000 for self-host\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets.\n\n # Image search tool (uses DuckDuckGo)\n # Use this to find reference images before image generation\n - name: image_search\n group: web\n use: deerflow.community.image_search.tools:image_search_tool\n max_results: 5\n\n # Image search tool (uses InfoQuest)\n # - name: image_search\n # group: web\n # use: deerflow.community.infoquest.tools:image_search_tool\n # # Used to limit the scope of image search results, only returns content within the specified time range. Set to -1 to disable time filtering\n # image_search_time_range: 10\n # # Image size filter. Options: \"l\" (large), \"m\" (medium), \"i\" (icon).\n # image_size: \"i\"\n\n # Image search tool (uses Serper - Google Images API, requires SERPER_API_KEY)\n # Serper provides real-time Google Images results. Sign up at https://serper.dev\n # Note: set SERPER_API_KEY in your environment before starting the app.\n # Avoid putting literal API keys in config.yaml; use the $VAR form instead.\n # - name: image_search\n # group: web\n # use: deerflow.community.serper.tools:image_search_tool\n # max_results: 5 # capped at 10 by the Serper provider\n # # api_key: $SERPER_API_KEY # Optional explicit env-var reference\n\n # Image search tool (uses Brave Image Search API, requires BRAVE_SEARCH_API_KEY)\n # Brave provides independent image results and works alongside the Brave web search tool.\n # Note: set BRAVE_SEARCH_API_KEY in your environment before starting the app.\n # Avoid putting literal API keys in config.yaml; use the $VAR form instead.\n # - name: image_search\n # group: web\n # use: deerflow.community.brave.tools:image_search_tool\n # max_results: 5 # capped at 200 by Brave Image Search\n # # country: US\n # # search_lang: en\n # # safesearch: strict\n # # spellcheck: true\n # # api_key: $BRAVE_SEARCH_API_KEY # Optional explicit env-var reference\n\n # File operations tools\n - name: ls\n group: file:read\n use: deerflow.sandbox.tools:ls_tool\n\n - name: read_file\n group: file:read\n use: deerflow.sandbox.tools:read_file_tool\n\n - name: glob\n group: file:read\n use: deerflow.sandbox.tools:glob_tool\n max_results: 200\n\n - name: grep\n group: file:read\n use: deerflow.sandbox.tools:grep_tool\n max_results: 100\n\n - name: write_file\n group: file:write\n use: deerflow.sandbox.tools:write_file_tool\n\n - name: str_replace\n group: file:write\n use: deerflow.sandbox.tools:str_replace_tool\n\n # Bash execution tool\n # Active only when using an isolated shell sandbox or when\n # sandbox.allow_host_bash: true explicitly opts into host bash.\n - name: bash\n group: bash\n use: deerflow.sandbox.tools:bash_tool\n\n# ============================================================================\n# Tool Search Configuration (Deferred Tool Loading)\n# ============================================================================\n# When enabled, MCP tools are not loaded into the agent's context directly.\n# Instead, they are listed by name in the system prompt and discoverable\n# via the tool_search tool at runtime.\n# This reduces context usage and improves tool selection accuracy when\n# multiple MCP servers expose a large number of tools.\n\ntool_search:\n enabled: false\n # When tool_search is enabled, PR1 MCP routing metadata can auto-promote\n # matching deferred MCP tool schemas before a model call. This is the maximum\n # number of matched schemas promoted per model call. Valid range: 1..5.\n auto_promote_top_k: 3\n\n# ============================================================================\n# Tool Output Budget Protection\n# ============================================================================\n# Prevents oversized tool results from blowing the model context window.\n# Outputs exceeding `externalize_min_chars` are persisted to disk and replaced\n# with a compact typed synopsis + file reference. The model can read the full output\n# via read_file. When disk persistence is unavailable, outputs exceeding\n# `fallback_max_chars` are head+tail truncated instead.\n#\n# `exempt_tools` prevents persist→read→persist infinite loops for read tools.\n# `tool_overrides` allows per-tool threshold customization.\n\ntool_output:\n enabled: true\n externalize_min_chars: 12000\n # Sampling budget for the inline raw head/tail sample appended to every\n # typed synopsis; ignored for binary-like output, which carries its own sample.\n preview_head_chars: 2000\n preview_tail_chars: 1000\n fallback_max_chars: 30000\n fallback_head_chars: 8000\n fallback_tail_chars: 3000\n storage_subdir: \".tool-results\"\n exempt_tools:\n - read_file\n - read_file_tool\n # tool_overrides:\n # web_search: 8000\n # bash: 20000\n\n# ============================================================================\n# Suggestions Configuration\n# ============================================================================\n# Configure whether the agent automatically generates follow-up question\n# suggestions at the end of each response.\n\nsuggestions:\n enabled: true\n max_suggestions: 3\n\n\n# ============================================================================\n# Input Polish Configuration\n# ============================================================================\n# Configure whether the composer can rewrite draft input before sending.\n\ninput_polish:\n enabled: true\n # Maximum draft length accepted by /api/input-polish.\n max_chars: 4000\n # Optional fast model for draft polishing. Leave null to use the default chat model.\n # For best UX, set this to your lowest-latency inexpensive model.\n model_name: null\n\n\n# ============================================================================\n# Loop Detection Configuration\n# ============================================================================\n# Detect and interrupt repeated identical tool-call loops.\n# Frequency thresholds are safety limits for repeated use of the same tool type.\n\nloop_detection:\n enabled: true\n warn_threshold: 3\n hard_limit: 5\n window_size: 20\n max_tracked_threads: 100\n tool_freq_warn: 30\n tool_freq_hard_limit: 50\n # Per-tool overrides for tool_freq_warn / tool_freq_hard_limit. Values can be\n # higher or lower than the global defaults. Commonly used to raise thresholds\n # for high-frequency tools like bash in batch workflows (e.g. RNA-seq pipelines)\n # without weakening protection on every other tool.\n # tool_freq_overrides:\n # bash:\n # warn: 150\n # hard_limit: 300\n\n# ============================================================================\n# Tool Progress State Machine Configuration (RFC #3177)\n# ============================================================================\n# Detects tool stagnation and repetition at the (thread, tool) level.\n# Tracks consecutive \"no-new-info\" calls (error, partial_success, near-duplicate success).\n# Three transition paths (determined by deerflow_tool_meta.recoverable_by_model):\n# recoverable=true (no_results, not_found, permission): ACTIVE → WARNED (terminal; hint re-injected each call)\n# recoverable=false (rate_limited, transient): ACTIVE → WARNED → BLOCKED after warn_escalation_count more\n# recoverable=false + action=stop (auth, config): ACTIVE → BLOCKED immediately\n# Requires ToolErrorHandlingMiddleware to be active (always on).\n\n# tool_progress:\n# enabled: false\n# stagnation_threshold: 3 # Consecutive problems before WARNED\n# warn_escalation_count: 2 # More problems after WARNED before BLOCKED\n# inject_assessment: true\n# jaccard_similarity_threshold: 0.8 # Word-set similarity threshold for near-duplicate detection\n# min_word_count_for_similarity: 10 # Min unique words to apply Jaccard check\n# max_tracked_threads: 100\n# exempt_tools:\n# - ask_clarification\n# - write_todos\n# - present_files\n# - task\n\n# ============================================================================\n# Read-Before-Write File Gate (issue #3857)\n# ============================================================================\n# Blocks write_file (append / overwrite of an existing file) and str_replace\n# unless the agent has read the file's current version first; any write\n# invalidates earlier reads, forcing a re-read between consecutive edits.\n# Deterministic guardrail against blind duplicate appends in long tasks.\n\nread_before_write:\n enabled: true\n\n# ============================================================================\n# Provider Safety Termination Configuration\n# ============================================================================\n# Intercept AIMessages where the provider stopped generation for safety reasons\n# (e.g. OpenAI finish_reason='content_filter', Anthropic stop_reason='refusal',\n# Gemini finish_reason='SAFETY') while still returning tool_calls. The\n# tool_calls in such responses are typically truncated/unreliable and must\n# not be executed. See issue #3028 for the full failure mode.\n#\n# Detectors are loaded by class path via reflection (same pattern as\n# guardrails / models / tools). The built-in set covers OpenAI-compatible\n# content_filter, Anthropic refusal, and Gemini SAFETY/BLOCKLIST/\n# PROHIBITED_CONTENT/SPII/RECITATION.\n\nsafety_finish_reason:\n enabled: true\n # Leave `detectors` unset to use the built-in detector set. Set to a\n # non-empty list to fully override (use `enabled: false` to disable instead\n # of providing an empty list).\n #\n # Example — extend the OpenAI-compatible detector for a Chinese provider\n # whose gateway uses a non-standard finish_reason token:\n # detectors:\n # - use: deerflow.agents.middlewares.safety_termination_detectors:OpenAICompatibleContentFilterDetector\n # config:\n # finish_reasons: [\"content_filter\", \"sensitive\", \"risk_control\"]\n # - use: deerflow.agents.middlewares.safety_termination_detectors:AnthropicRefusalDetector\n # - use: deerflow.agents.middlewares.safety_termination_detectors:GeminiSafetyDetector\n #\n # Example — add a custom detector for an in-house provider:\n # detectors:\n # - use: my_company.deerflow_ext:WenxinSafetyDetector\n # config:\n # error_codes: [336003, 17, 18]\n\n# ============================================================================\n# Sandbox Configuration\n# ============================================================================\n# Choose between local sandbox (direct execution) or Docker-based AIO sandbox\n\n# Option 1: Local Sandbox (Default)\n# Executes commands directly on the host machine\nuploads:\n # Application-level upload limits enforced by the gateway and exposed to the\n # frontend before file selection.\n max_files: 10\n max_file_size: 52428800 # 50 MiB\n max_total_size: 104857600 # 100 MiB\n # Automatic Office/PDF conversion runs on the backend host before sandbox\n # isolation applies. Keep this disabled unless uploads come from a fully\n # trusted source and you intentionally accept host-side parser risk.\n auto_convert_documents: false\n # Controls which PDF-to-Markdown converter is used whenever PDF conversion\n # runs. Automatic upload conversion is gated separately by\n # auto_convert_documents.\n # auto — prefer pymupdf4llm when installed; fall back to MarkItDown for\n # image-based or encrypted PDFs (recommended default).\n # pymupdf4llm — always use pymupdf4llm (must be installed: uv add pymupdf4llm).\n # Better heading/table extraction; faster on most files.\n # markitdown — always use MarkItDown (original behaviour, no extra dependency).\n pdf_converter: auto\n\nsandbox:\n use: deerflow.sandbox.local:LocalSandboxProvider\n # Host bash execution is disabled by default because LocalSandboxProvider is\n # not a secure isolation boundary for shell access. Enable only for fully\n # trusted, single-user local workflows.\n allow_host_bash: false\n # Optional: Mount additional host directories into the sandbox.\n # Each mount maps a host path to a virtual container path accessible by the agent.\n # Note: with LocalSandboxProvider under `make up` (docker-compose), host_path is\n # checked from inside the deer-flow-gateway container — you must also bind-mount\n # the same directory into services.gateway.volumes in docker/docker-compose.yaml\n # for this mount to take effect (see issue #3244).\n # mounts:\n # - host_path: /home/user/my-project # Absolute path; see note above for Docker mode\n # container_path: /mnt/my-project # Virtual path inside the sandbox\n # read_only: true # Whether the mount is read-only (default: false)\n\n # Tool output truncation limits (characters).\n # bash uses middle-truncation (head + tail) since errors can appear anywhere in the output.\n # read_file and ls use head-truncation since their content is front-loaded.\n # Set to 0 to disable truncation.\n bash_output_max_chars: 20000\n read_file_output_max_chars: 50000\n ls_output_max_chars: 20000\n\n # Maximum wall-clock seconds a single host bash command may run before it is\n # terminated (process group and all). A blocking foreground command — e.g. a\n # server started without backgrounding — is killed after this long so the\n # agent's turn cannot hang. Start long-lived processes in the background with\n # output redirected (e.g. `your-command > /tmp/server.log 2>&1 &`) when you\n # need logs; unredirected background output is drained with bounded capture\n # and excess output is discarded.\n bash_command_timeout: 600\n\n# Option 2: Container-based AIO Sandbox\n# Executes commands in isolated containers (Docker or Apple Container)\n# On macOS: Automatically prefers Apple Container if available, falls back to Docker\n# On other platforms: Uses Docker\n# Uncomment to use:\n# sandbox:\n# use: deerflow.community.aio_sandbox:AioSandboxProvider\n#\n# # Optional: Container image to use (works with both Docker and Apple Container)\n# # Default: enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest\n# # The mirror's `:latest` tag is frozen on an old pre-1.9.3 digest that lacks\n# # the /v1/bash/* routes required-secrets skills need (see #3921/#3922), so\n# # pin an explicit version instead — recommended: 1.11.0 (multi-arch, works\n# # on both x86_64 and arm64). Custom images should extend the default image\n# # or implement the same AIO sandbox HTTP API used by agent-sandbox. See\n# # backend/docs/CONFIGURATION.md.\n# # image: enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0\n#\n# # Optional: Base port for sandbox containers (default: 8080)\n# # port: 8080\n\n# # Optional: Maximum number of concurrent sandbox containers (default: 3)\n# # When the limit is reached the least-recently-used sandbox is evicted to\n# # make room for new ones. Use a positive integer here; omit this field to use the default.\n# # replicas: 3\n#\n# # Optional: Prefix for container names (default: deer-flow-sandbox)\n# # container_prefix: deer-flow-sandbox\n#\n# # Optional: Override whether the sandbox already sees the gateway's\n# # thread workspace/uploads/outputs through shared mounts.\n# # Omit this field to auto-detect from the backend (local containers: true;\n# # remote/provisioner backends: false). Set true only when the deployment\n# # guarantees both sides use the same thread user-data directories, such as\n# # a correctly aligned shared PVC, NFS volume, hostPath, or bind mount.\n# # true skips per-upload sandbox acquire/sync; false forces explicit sync.\n# # thread_data_mounts: true\n#\n# # Optional: Additional mount directories from host to container\n# # NOTE: Skills directory is automatically mounted from skills.path to skills.container_path\n# # mounts:\n# # # Other custom mounts\n# # - host_path: /path/on/host\n# # container_path: /home/user/shared\n# # read_only: false\n# #\n# # # DeerFlow will surface configured container_path values to the agent,\n# # # so it can directly read/write mounted directories such as /home/user/shared\n#\n# # Optional: Environment variables to inject into the sandbox container\n# # Values starting with $ will be resolved from host environment variables\n# # environment:\n# # NODE_ENV: production\n# # DEBUG: \"false\"\n# # API_KEY: $MY_API_KEY # Reads from host's MY_API_KEY env var\n# # DATABASE_URL: $DATABASE_URL # Reads from host's DATABASE_URL env var\n#\n# # Optional: Cross-instance container ownership (issue #4206).\n# #\n# # Gateway instances share sandbox containers but each keeps its own in-memory\n# # warm pool. Without shared ownership state, one instance's startup\n# # reconciliation adopts a container another instance is actively using and\n# # later idle-destroys it — tool calls then fail with 502 / connection refused.\n# #\n# # Single gateway instance? Leave this out; `memory` is the default and the\n# # cross-instance kill cannot happen.\n# #\n# # MULTIPLE gateway instances / workers sharing one container backend\n# # (load-balanced deployments, Docker Compose) MUST set type: redis. Docker\n# # Compose already sets DEER_FLOW_STREAM_BRIDGE_REDIS_URL, which is taken as\n# # proof the deployment is multi-instance, so redis ownership is inferred even\n# # if this section is omitted.\n# #\n# # The redis ownership store requires the optional `redis` extra. It is\n# # auto-detected from this section on `make dev` and always installed in the\n# # Docker image. To install it manually:\n# # cd backend && uv sync --all-packages --extra redis\n# #\n# # ownership:\n# # type: memory # single gateway instance only\n# #\n# # ownership:\n# # type: redis # required for multi-instance / load-balanced\n# # redis_url: redis://redis:6379/0\n# # renewal_interval_seconds: 30 # how often an owner refreshes its leases\n# # ttl_multiplier: 4 # lease TTL = interval x this (min 2, so a\n# # # single missed renewal cannot expire a live\n# # # owner). Liveness is deliberately independent\n# # # of idle_timeout: renewal keeps running even\n# # # at idle_timeout: 0.\n# # key_prefix: deerflow:sandbox:owner\n# #\n# # NOTE: the redis ownership store is fail-closed, matching the stream bridge's\n# # fail-hard policy. Redis.from_url is lazy so a down Redis does not block\n# # startup, but a sandbox whose ownership cannot be published is not handed out\n# # — acquiring raises instead. The alternative (proceed unowned) is exactly the\n# # #4206 cross-instance kill. Run Redis with HA / a restart policy.\n# #\n# # NOTE: the other boundary is the lease TTL (renewal_interval_seconds x\n# # ttl_multiplier). A Redis outage longer than the TTL can let a reconciling\n# # instance adopt a live owner's container: a lapsed lease is indistinguishable\n# # from a dead owner, so one TTL of adoption grace is the whole safety margin.\n# # Size the TTL against your Redis availability target (HA Redis keeps this\n# # window out of reach).\n\n# Option 3: BoxLite micro-VM Sandbox\n# Runs each sandbox as a BoxLite micro-VM. Released boxes stay in an in-process\n# warm pool and can be reclaimed by the same user/thread without a cold start.\n# Requires the boxlite runtime and host virtualization support (KVM on Linux,\n# Hypervisor.framework on macOS).\n# sandbox:\n# use: deerflow.community.boxlite:BoxliteProvider\n# image: python:3.12-slim\n#\n# # Optional: Per-box memory and CPU limits.\n# # memory_mib: 1024\n# # cpus: 2\n#\n# # Optional: Maximum active + warm BoxLite VMs per gateway process (default: 3).\n# # Active boxes are never evicted; only warm-pool boxes are stopped to make room.\n# # replicas: 3\n#\n# # Optional: Seconds before an idle warm-pool VM is stopped (default: 600).\n# # Set to 0 to keep warm VMs until shutdown or replica eviction.\n# # idle_timeout: 600\n#\n# # Optional: Skip the reclaim health check for very recently released VMs.\n# # Default 0.0 keeps reliability-first validation before warm reuse.\n# # health_check_skip_seconds: 0.0\n#\n# # Optional: Environment variables to inject into every command.\n# # environment:\n# # PYTHONUNBUFFERED: \"1\"\n#\n# Option 4: Provisioner-managed AIO Sandbox (docker-compose-dev)\n# Each sandbox_id gets a dedicated Pod in k3s, managed by the provisioner.\n# Recommended for production or advanced users who want better isolation and scalability.:\n# sandbox:\n# use: deerflow.community.aio_sandbox:AioSandboxProvider\n# provisioner_url: http://provisioner:8002\n# # API key for provisioner authentication. Must match PROVISIONER_API_KEY\n# # set on the provisioner container. Both sides must have the same value set;\n# # the provisioner rejects all /api/* requests when PROVISIONER_API_KEY is unset.\n# # Generate a strong key: openssl rand -hex 32\n# # provisioner_api_key: $PROVISIONER_API_KEY\n# # Note: provisioner-created Pods use the provisioner's SANDBOX_IMAGE\n# # environment variable, not sandbox.image from this config file.\n\n# Option 5: Tenki cloud microVM Sandbox\n# Runs each sandbox as an isolated Tenki cloud microVM. Released sandboxes stay\n# in an in-process warm pool and are reclaimed by the same user/thread without a\n# cold start. Requires the optional SDK: pip install \"deerflow-harness[tenki]\".\n# sandbox:\n# use: deerflow.community.tenki:TenkiSandboxProvider\n# # api_key: $TENKI_API_KEY # falls back to TENKI_API_KEY / TENKI_AUTH_TOKEN env var\n# # base_url: https://tenki.cloud # optional; SDK default when omitted\n# # image: my-base-image # optional; Tenki account default base image when omitted\n# # project_id: proj_... # optional; auto-selected if the account has exactly one\n# # workspace_id: ws_... # optional; auto-selected if the account has exactly one\n# # cpu_cores: 2 # optional per-sandbox vCPUs\n# # memory_mb: 2048 # optional per-sandbox memory\n# # replicas: 3 # active + warm microVM cap per gateway process\n# # idle_timeout: 600 # warm microVM idle seconds before terminate; 0 disables\n# # max_duration: 14400 # Tenki sandbox lifetime in seconds; 0 uses the account default\n# # sticky: false # pin the microVM to its host (only matters with pause/resume)\n# # home_dir: /home/tenki # writable dir backing /mnt/user-data\n# # environment: # injected into every command (and as create-time env)\n# # PYTHONUNBUFFERED: \"1\"\n\n# ============================================================================\n# Subagents Configuration\n# ============================================================================\n# Configure timeouts for subagent execution\n# Subagents are background workers delegated tasks by the lead agent\n\n# subagents:\n# # Default timeout (seconds) for built-in subagents (default: 1800 = 30 min).\n# # Custom agents use their own timeout_seconds (default 900) unless overridden.\n# timeout_seconds: 1800\n# # Optional global max-turn override for all subagents.\n# # Built-in defaults: general-purpose=150, bash=60. Leave unset to keep them.\n# # max_turns: 120\n#\n# # Total number of subagent delegations allowed in one lead-agent run.\n# # This is a deterministic backstop against repeated planning checkpoints\n# # launching legal-sized batches forever. The default 6 allows two full\n# # batches at the default concurrency of 3. Valid config range: 1-50.\n# # Per-request runtime context can temporarily override this with\n# # `max_total_subagents`, clamped to the same 1-50 range.\n# max_total_per_run: 6\n#\n# # Per-run token ceiling for subagents (#3875 Phase 2). A backstop against a\n# # subagent that burns tokens on trivial work. At the hard-stop threshold the\n# # in-flight turn is capped (tool calls stripped, finish_reason forced to\n# # \"stop\") so the run completes naturally with a final answer; the result is\n# # stamped `completed` + `subagent_stop_reason=token_capped` so the lead and\n# # UI can tell a budget-capped completion from a clean one. The 2,000,000\n# # default is a generous ceiling — lower it to tighten cost controls. A\n# # per-agent `token_budget` override (see `agents:` below) wins over this.\n# # token_budget:\n# # enabled: true\n# # max_tokens: 2000000\n# # warn_threshold: 0.7 # log a warning once this fraction of the budget is spent\n#\n# # Optional per-agent overrides (applies to both built-in and custom agents)\n# agents:\n# general-purpose:\n# timeout_seconds: 2700 # 45 minutes for very long deep-research tasks\n# max_turns: 250 # raise above the 150 default for very deep tasks\n# # token_budget: # per-agent override of the global token_budget above\n# # max_tokens: 3000000 # raise the ceiling for deep-research tasks\n# # model: qwen3:32b # Use a specific model (default: inherit from lead agent)\n# # skills: # Skill discovery/activation allowlist (default: all enabled)\n# # - web-search\n# # - data-analysis\n# bash:\n# timeout_seconds: 300 # 5 minutes for quick command execution\n# max_turns: 80\n# # skills: [] # No discoverable/activatable skills for bash agent\n#\n# # Custom subagent types: define specialized agents with their own prompts,\n# # tools, skills, and model configuration. Custom agents are available via\n# # the `task` tool alongside built-in types (general-purpose, bash).\n# # custom_agents:\n# # analysis:\n# # description: \"Data analysis specialist for processing datasets and generating insights\"\n# # system_prompt: |\n# # You are a data analysis subagent. Focus on:\n# # - Processing and analyzing datasets\n# # - Generating visualizations\n# # - Providing statistical insights\n# # tools: # Tool whitelist (null = inherit all)\n# # - bash\n# # - read_file\n# # - write_file\n# # skills: # Skill discovery/activation allowlist (null = all, [] = none)\n# # - data-analysis\n# # - visualization\n# # model: inherit # 'inherit' uses parent's model\n# # max_turns: 80\n# # timeout_seconds: 600\n#\n# # Model override: by default, subagents inherit the lead agent's model.\n# # Set `model` to use a different model (e.g., a local Ollama model for cost savings).\n# # The model name must match a name defined in the `models:` section above.\n\n# ============================================================================\n# ACP Agents Configuration\n# ============================================================================\n# Configure external ACP-compatible agents for the built-in `invoke_acp_agent` tool.\n\n# acp_agents:\n# claude_code:\n# # DeerFlow expects an ACP adapter here. The standard `claude` CLI does not\n# # speak ACP directly. Install `claude-agent-acp` separately or use:\n# command: npx\n# args: [\"-y\", \"@zed-industries/claude-agent-acp\"]\n# description: Claude Code for implementation, refactoring, and debugging\n# model: null\n# # auto_approve_permissions: false # Set to true to auto-approve ACP permission requests\n# # timeout_seconds: 1800 # Abort + kill the subprocess if it doesn't respond in time (default: 1800 = 30 min)\n# # env: # Optional: inject environment variables into the agent subprocess\n# # ANTHROPIC_API_KEY: $ANTHROPIC_API_KEY # $VAR resolves from host environment\n#\n# codex:\n# # DeerFlow expects an ACP adapter here. The standard `codex` CLI does not\n# # speak ACP directly. Install `codex-acp` separately or use:\n# command: npx\n# args: [\"-y\", \"@zed-industries/codex-acp\"]\n# description: Codex CLI for repository tasks and code generation\n# model: null\n# # auto_approve_permissions: false # Set to true to auto-approve ACP permission requests\n# # env: # Optional: inject environment variables into the agent subprocess\n# # OPENAI_API_KEY: $OPENAI_API_KEY # $VAR resolves from host environment\n\n# ============================================================================\n# Skills Configuration\n# ============================================================================\n# Configure skills directory for specialized agent workflows\n\nskills:\n # Path to skills directory on the host (relative to project root or absolute)\n # Default: skills under the project root\n # Override with DEER_FLOW_SKILLS_PATH when this field is omitted.\n # Uncomment to customize:\n # path: /absolute/path/to/custom/skills\n\n # Path where skills are mounted in the sandbox container\n # This is used by the agent to access skills in both local and Docker sandbox\n # Default: /mnt/skills\n container_path: /mnt/skills\n\n # Deferred skill discovery (default: false)\n # When enabled, only skill names appear in the system prompt ().\n # The LLM discovers skill details on demand via the describe_skill tool.\n # This keeps the system prompt compact and prefix-cache friendly when many\n # skills are installed.\n # deferred_discovery: true\n\n# ============================================================================\n# SkillScan Configuration\n# ============================================================================\n# Native deterministic skill safety scanning. This runs before the LLM skill\n# scanner on skill install/update and agent-managed skill writes.\nskill_scan:\n # Set false to disable the new deterministic analyzers (nested-archive,\n # secret-pattern, and other content-level checks). Safe archive extraction\n # (path traversal, symlinks, executable-binary, total-size, and entry-count\n # limits) and the LLM skill scanner still run unconditionally.\n enabled: true\n\n# Note: To restrict which skills are loaded for a specific custom agent,\n# define a `skills` list in that agent's `config.yaml` (e.g. `agents/my-agent/config.yaml`):\n# - Omitted or null: load all globally enabled skills (default)\n# - []: disable all skills for this agent\n# - [\"skill-name\"]: load only specific skills\n\n# ============================================================================\n# Title Generation Configuration\n# ============================================================================\n# Automatic conversation title generation settings\n\ntitle:\n enabled: true\n max_words: 6\n max_chars: 60\n model_name: null # null = fast local fallback; set a model name to use LLM title generation\n\n# ============================================================================\n# Summarization Configuration\n# ============================================================================\n# Automatically summarize conversation history when token limits are approached\n# This helps maintain context in long conversations without exceeding model limits\n\nsummarization:\n enabled: true\n\n # Model to use for summarization.\n # null = summarize with the model the run actually uses (the lead run's model, a\n # subagent's own model, or a thread's custom-agent model), NOT models[0].\n # set = that model generates; if its provider fails, compaction falls back to the\n # run's own model so a broken summary provider cannot disable compaction.\n # Recommended: Use a lightweight, cost-effective model like \"gpt-4o-mini\" or similar\n model_name: null\n\n # Trigger conditions - at least one required\n # Summarization runs when ANY threshold is met (OR logic)\n # You can specify a single trigger or a list of triggers\n trigger:\n # Trigger when token count reaches 32000\n - type: tokens\n value: 32000\n # Uncomment to also trigger when message count reaches 50\n # - type: messages\n # value: 50\n # Uncomment to trigger when 80% of model's max input tokens is reached\n # - type: fraction\n # value: 0.8\n\n # Context retention policy after summarization\n # Specifies how much recent history to preserve\n keep:\n # Keep the most recent 10 messages (recommended)\n type: messages\n value: 10\n # Alternative: Keep specific token count\n # type: tokens\n # value: 3000\n # Alternative: Keep percentage of model's max input tokens\n # type: fraction\n # value: 0.3\n\n # Maximum tokens to keep when preparing messages for summarization\n # Set to null to skip trimming (not recommended for very long conversations)\n trim_tokens_to_summarize: 15564\n\n # Custom summary prompt template (null = use default LangChain prompt)\n # The prompt should guide the model to extract important context\n summary_prompt: null\n\n # Loaded SKILL.md references (read_file calls under skills.container_path) are\n # captured into the durable skill_context channel and re-injected after\n # compaction as name/path/description reminders. The full skill body is not\n # persisted; the agent should re-read the file before applying instructions.\n # Tool names counted as skill reads:\n # Legacy preserve_recent_skill_* summarization settings are no longer used;\n # skill retention is handled by this durable reference channel instead. Set\n # this list to [] to disable durable skill-reference capture.\n skill_file_read_tool_names:\n - read_file\n - read\n - view\n - cat\n\n# ============================================================================\n# Memory Configuration\n# ============================================================================\n# Global memory mechanism (pluggable + self-contained).\n#\n# Shared fields (host level, backend-agnostic):\n# enabled - Master switch for the memory mechanism (call-site gate)\n# injection_enabled - Whether to inject memory into the system prompt (call-site gate)\n# shutdown_flush_timeout_seconds - Hard budget (s) to drain pending updates on Gateway graceful shutdown (default: 30)\n# manager_class - Backend selector: registered name (deermem/mem0/noop/openviking) or dotted path\n# backend_config - Backend-private config dict (passthrough; each backend self-interprets)\n#\n# DeerMem-private fields live under ``backend_config`` (NOT at the memory: top level):\n# storage_path - Data root. Empty = deer-flow base_dir (factory injects absolute\n# runtime_home). Absolute path = that root. Relative = CWD-relative.\n# model - LLM config for memory extraction: {provider, model, api_key, base_url,\n# temperature}. Omit all fields = no extraction (non-LLM ops still work).\n# debounce_seconds - Debounce wait before processing queued updates (default: 30)\n# max_facts - Maximum facts to store (default: 100)\n# fact_confidence_threshold - Minimum confidence for storing facts (default: 0.7)\n# max_injection_tokens - Token budget for memory injection (default: 2000)\n# token_counting - tiktoken (accurate, network on first use) or char (network-free)\n# guaranteed_categories - Fact categories always injected (default: [\"correction\"])\n# guaranteed_token_budget - Token ceiling for guaranteed categories (default: 500)\n# staleness_review_enabled - Enable staleness pruning (default: true)\n# staleness_age_days - Age threshold for staleness candidates (default: 90)\n# staleness_min_candidates - Minimum stale facts to trigger review (default: 3)\n# staleness_max_removals_per_cycle - Safety cap on removals per cycle (default: 10)\n# staleness_protected_categories - Categories exempt from staleness review (default: [\"correction\"])\n# staleness_max_lifetime_multiplier - Creation-time cap multiplier for a fact LLM-assigned\n# expected_valid_days; new facts clamped to\n# staleness_age_days x multiplier. Default 20.0\n# (90 x 20 = 1800d ~ 5 years) supports the very-stable\n# prompt tier. (default: 20.0, range: 1.0-100.0)\n# staleness_max_extension_days - Absolute ceiling (in days) on expected_valid_days after a\n# lifetime extension (staleFactsToExtend). Prevents timedelta\n# overflow and LLM misfire from permanently deferring a fact.\n# (default: 3650, range: 90-36500)\nmemory:\n enabled: true\n injection_enabled: true # Whether to inject memory into system prompt\n # Hard budget (seconds) to drain pending memory updates on Gateway graceful\n # shutdown. Each pending item does one LLM call, so large IM batches may need\n # more. Must fit inside the pod's K8s terminationGracePeriodSeconds (channel\n # stop + this drain + buffer) or K8s SIGKILLs the drain -- set that on the\n # gateway Helm deployment (see deploy/helm/deer-flow). Default 30s.\n shutdown_flush_timeout_seconds: 30.0\n # Memory backend selector. Either a registered backend name (matching a\n # backends// folder that exposes MANAGER_CLASS, e.g. deermem / mem0 / noop)\n # or a dotted import path to a MemoryManager subclass.\n manager_class: deermem\n # Memory operation mode:\n # middleware (default) - passive background extraction after each turn.\n # tool - experimental opt-in; the model calls memory_search/memory_add/\n # memory_update/memory_delete directly. This gives the model agency over\n # memory writes, but effectiveness depends on model tool-use behavior.\n # Normally only one mode runs at a time. A backend that needs conversation-\n # level extraction may retain passive writes in tool mode while still\n # exposing query-aware search (mem0 does this). Tool calls go through the\n # active MemoryManager; unsupported fact CRUD returns a JSON error.\n mode: middleware\n # Backend-private config (a dict), passed verbatim to the backend __init__.\n # Each backend self-interprets it (DeerMem parses it into DeerMemConfig).\n #\n # OpenViking HTTP example (replace this DeerMem backend_config block when\n # manager_class is openviking; OpenViking currently supports middleware mode):\n #\n # backend_config:\n # base_url: http://openviking:1933\n # auth_mode: trusted\n # account: deerflow\n # api_key_env: OPENVIKING_API_KEY\n # max_connections: 100\n # max_keepalive_connections: 20\n # max_seen_message_ids: 512 # recent-ID fallback for compacted histories\n # startup_policy: fail_fast\n # failure_policy:\n # read: fail_open\n # write: log_and_drop\n # retrieval:\n # top_k: 8\n # score_threshold: 0.25\n # max_injection_chars: 12000\n #\n # For a host-installed OpenViking used by Docker DeerFlow, set base_url to\n # http://host.docker.internal:1933 and allow_insecure_http: true. The bundled\n # optional Compose overlay uses the internal http://openviking:1933 address.\n backend_config:\n storage_path: \"\" # empty = deer-flow base_dir (factory injects absolute runtime_home); a non-empty path is the root DIRECTORY (per-user memory under {storage_path}/users/{uid}/memory.json)\n storage_class: file # file (default) or a dotted MemoryStorage class path; invalid persistent backends fail fast\n strict_user_scope: false # set true in authenticated deployments after all callers propagate user_id\n manifest_filename: memory.json # user-global JSON: version/revision/time + user/history only; no facts or fact index\n file_lock_timeout_seconds: 10 # per-scope cross-process advisory lock timeout (single-machine local filesystem)\n retrieval_adapter: fts5 # fts5 (default), empty to disable, or a dotted RetrievalPort factory(config)\n debounce_seconds: 30 # Wait time before processing queued updates\n # Backpressure cap on pending items. 0 = unlimited. At the cap, new\n # non-signal updates are rejected (QueueFull); signal updates are always\n # admitted so important memories are never shed.\n queue_max_depth: 1000\n model: # LLM for memory extraction; omit all fields = no extraction (non-LLM ops still work; an update raises)\n # provider: openai\n # model: gpt-4o-mini\n # api_key: $OPENAI_API_KEY\n # base_url: # optional, for OpenAI-compatible gateways (e.g. DeepSeek)\n # temperature: # optional\n max_facts: 100 # Maximum number of facts to store\n fact_confidence_threshold: 0.7 # Minimum confidence for storing facts\n max_injection_tokens: 2000 # Maximum tokens for memory injection\n # Token counting strategy for memory-injection budgeting:\n # tiktoken (default) - accurate, but the encoding BPE data may be\n # downloaded from a public network endpoint on first use. In\n # network-restricted environments this download can block for a long\n # time (see issues #3402 / #3429). Pre-cache the encoding or set this\n # to \"char\" to avoid it.\n # char - network-free CJK-aware character-based estimate; never touches\n # tiktoken. Slightly less precise budgeting, zero network I/O.\n token_counting: tiktoken\n # Guaranteed injection: fact categories that bypass the regular token budget\n # and draw from a reserved allowance, so high-signal corrections (e.g.\n # \"don use pip, use uv\") survive even when the budget is tight.\n guaranteed_categories:\n - correction\n guaranteed_token_budget: 500\n # Staleness review: periodically prune aged facts that may no longer reflect\n # the user current situation. When triggered, the LLM reviews facts older\n # than their individual review window (expected_valid_days, or\n # staleness_age_days as fallback) during the normal memory-update call (same\n # LLM invocation - no extra API call) and decides KEEP, REMOVE, or EXTEND\n # for each. The LLM assigns expected_valid_days when creating a fact; EXTEND\n # (staleFactsToExtend) recalibrates that window at review time.\n staleness_review_enabled: true\n staleness_age_days: 90\n staleness_min_candidates: 3\n staleness_max_removals_per_cycle: 10\n staleness_protected_categories:\n - correction\n staleness_max_lifetime_multiplier: 20.0 # creation cap = staleness_age_days x multiplier (90 x 20 = 1800d)\n staleness_max_extension_days: 3650 # absolute ceiling on extended expected_valid_days (~10 years)\n # Memory consolidation (opt-in, lossy: source facts are replaced by a\n # synthesized one; only consolidatedFrom IDs are kept). Runs in the same\n # memory-update LLM call as extraction/staleness - no extra API cost.\n # consolidation_enabled defaults to false because consolidation is lossy.\n consolidation_enabled: false # set true to opt into fact consolidation\n consolidation_min_facts: 8 # min facts in one category to trigger review (3-30)\n consolidation_max_groups_per_cycle: 3 # max groups merged per update cycle (1-10)\n consolidation_max_sources: 8 # max source facts per consolidation group (2-20)\n # extraction_callback is a host-injected post-extraction observability hook\n # (token usage, facts accepted/rejected, rejection rate). The factory\n # injects a logging default; set programmatically to emit a Langfuse span.\n # Message processing (externalized patterns / prompt templates):\n # patterns_dir - dir with correction.yaml / reinforcement.yaml overriding\n # the bundled signal-detection patterns. None (default) =\n # bundled core/message_patterns/. When explicitly set, both\n # files must exist (typos or missing mounts raise an error).\n # prompts_dir - dir with custom memory-extraction prompt templates\n # (memory_update.chat.yaml, staleness_review.yaml,\n # consolidation.yaml, fact_extraction.yaml). None (default)\n # = bundled core/prompts/. Supports per-agent subdirectories.\n # patterns_dir: \"\" # empty = bundled defaults\n # prompts_dir: \"\" # empty = bundled defaults\n\n# ============================================================================\n# Custom Agent Management API\n# ============================================================================\n# Controls whether the HTTP gateway exposes custom-agent SOUL/USER.md management.\n# Keep this disabled unless the gateway is behind a trusted authenticated admin boundary.\nagents_api:\n enabled: false\n\n# ============================================================================\n# Skill Self-Evolution Configuration\n# ============================================================================\n# Allow the agent to autonomously create and improve skills in skills/custom/.\nskill_evolution:\n enabled: false # Set to true to allow agent-managed writes under skills/custom\n moderation_model_name: null # Model for LLM-based security scanning (null = use default model)\n security_fail_closed: true # Moderation model unavailable: true blocks all writes; false allows non-executable content with a warning (executable is always blocked)\n\n# ============================================================================\n# Checkpointer Configuration (DEPRECATED — use `database` instead)\n# ============================================================================\n# Legacy standalone checkpointer config. Kept for backward compatibility.\n# Prefer the unified `database` section below, which drives the LangGraph\n# checkpointer, LangGraph Store, and DeerFlow application data (runs,\n# feedback, events) from a single backend setting.\n#\n# If both `checkpointer` and `database` are present, `checkpointer`\n# takes precedence for the LangGraph checkpointer and Store only.\n#\n# checkpointer:\n# type: sqlite\n# connection_string: checkpoints.db\n#\n# checkpointer:\n# type: postgres\n# connection_string: postgresql://user:password@localhost:5432/deerflow\n# # Optional: place LangGraph checkpointer/store tables in this schema.\n# # Leave empty to use the server default search_path, usually public.\n# postgres_schema: deerflow\n\n# ============================================================================\n# Database\n# ============================================================================\n# Unified storage backend for the LangGraph checkpointer, LangGraph Store,\n# and DeerFlow application data (runs, threads metadata, feedback, etc.).\n#\n# backend: memory -- No persistence, data lost on restart\n# backend: sqlite -- Single-node deployment, files in sqlite_dir\n# backend: postgres -- Production multi-node deployment\n#\n# If this section is omitted or empty in config.yaml, DeerFlow uses:\n# backend: sqlite\n# sqlite_dir: .deer-flow/data\n#\n# SQLite mode uses a single deerflow.db file with WAL journal mode\n# for the checkpointer, Store, and application data.\n#\n# Postgres mode: put your connection URL in .env as DATABASE_URL,\n# then reference it here with $DATABASE_URL.\n#\n# Install the driver — Issue #2754 fix lands `UV_EXTRAS` in every code path:\n# Local `make dev` auto-detects from `database.backend: postgres` below\n# and passes `--extra postgres` to `uv sync` on every restart, so\n# the extra is no longer wiped. To opt in explicitly (or layer\n# extras like `postgres,ollama`), set in project-root .env:\n# UV_EXTRAS=postgres\n# Docker dev `make docker-start` reads `UV_EXTRAS` from project-root .env via\n# `env_file`. Set:\n# UV_EXTRAS=postgres\n# Multiple extras (`postgres,ollama`) supported here too — see\n# docker/dev-entrypoint.sh.\n# Docker img build-arg `UV_EXTRAS=postgres,discord docker compose build`\n# supports comma- or whitespace-separated extras at build time\n# (backend/Dockerfile expands them to repeated `--extra` flags).\n#\n# First-time bootstrap (before `make dev`):\n# cd backend && uv sync --all-packages --extra postgres\n# (--all-packages propagates the extra into workspace members — see PR #2584)\n#\n# NOTE: When both `checkpointer` and `database` are configured,\n# `checkpointer` takes precedence for the LangGraph checkpointer and Store;\n# `database` still controls DeerFlow application data.\n# If you use `database`, you can remove the `checkpointer` section.\n# database:\n# backend: sqlite\n# sqlite_dir: .deer-flow/data\n#\n# database:\n# backend: postgres\n# postgres_url: $DATABASE_URL\n# # Optional: place DeerFlow ORM tables and LangGraph checkpointer/store\n# # tables in this schema. Leave empty to use the server default\n# # search_path, usually public.\n# postgres_schema: deerflow\n#\n# postgres_schema only takes effect for new tables. On startup DeerFlow runs\n# CREATE SCHEMA IF NOT EXISTS and pins the connection search_path; it never\n# moves existing tables, so third-party tables in `public` are left untouched.\n#\n# Migrating an EXISTING deployment from `public` to a dedicated schema:\n# 1. Stop the DeerFlow services.\n# 2. Move EVERY DeerFlow-owned table AND the Alembic state into the new schema\n# (run as the DB owner). Missing any table strands its rows: after restart\n# DeerFlow recreates an empty counterpart in the target schema and the old\n# rows stay invisible in `public`. Moving the ORM tables but leaving\n# `alembic_version` behind is especially dangerous -- bootstrap then treats\n# the target schema as unversioned, re-baselines it, and replays migrations.\n# CREATE SCHEMA IF NOT EXISTS deerflow;\n# -- Application ORM tables:\n# ALTER TABLE public.runs SET SCHEMA deerflow;\n# ALTER TABLE public.run_events SET SCHEMA deerflow;\n# ALTER TABLE public.threads_meta SET SCHEMA deerflow;\n# ALTER TABLE public.feedback SET SCHEMA deerflow;\n# ALTER TABLE public.users SET SCHEMA deerflow;\n# ALTER TABLE public.agents SET SCHEMA deerflow;\n# -- IM channel tables:\n# ALTER TABLE public.channel_connections SET SCHEMA deerflow;\n# ALTER TABLE public.channel_credentials SET SCHEMA deerflow;\n# ALTER TABLE public.channel_oauth_states SET SCHEMA deerflow;\n# ALTER TABLE public.channel_conversations SET SCHEMA deerflow;\n# -- Scheduled-task tables:\n# ALTER TABLE public.scheduled_tasks SET SCHEMA deerflow;\n# ALTER TABLE public.scheduled_task_runs SET SCHEMA deerflow;\n# -- Alembic migration state (REQUIRED -- see note above):\n# ALTER TABLE public.alembic_version SET SCHEMA deerflow;\n# The DeerFlow-owned set grows over time; discover any tables the list\n# above misses (as the DeerFlow DB owner/role) with:\n# SELECT table_name FROM information_schema.tables WHERE table_schema='public';\n# LangGraph checkpointer/store table names also vary by version -- list them\n# the same way, then ALTER ... SET SCHEMA deerflow for each (e.g. checkpoints,\n# checkpoint_blobs, checkpoint_writes, checkpoint_migrations, store,\n# store_migrations).\n# 3. Set postgres_schema: deerflow here and restart.\n# 4. Verify with `SHOW search_path;` and a smoke test (create a run, read\n# its history). The `public` schema should gain no new DeerFlow tables.\ndatabase:\n backend: sqlite\n sqlite_dir: .deer-flow/data\n\n# ============================================================================\n# Inbound webhook dedupe storage (cross-pod redelivery dedup — issue #4120)\n# ============================================================================\n# Where the ChannelManager records inbound webhook dedupe state.\n#\n# backend: auto -- (default) Postgres whenever database.backend=postgres,\n# including multi-replica K8s where each pod runs a single\n# worker but shares one DB. Otherwise an in-process memory\n# store (single-DB, single-pod).\n# NOTE: multi-replica deployments must use a Postgres DB;\n# if database.backend is sqlite/memory, 'auto' falls back to\n# the per-pod memory store and cross-pod dedupe is disabled.\n# backend: memory -- In-process store only. Per-pod: a webhook redelivered to\n# a DIFFERENT replica is NOT deduped. Not recommended for\n# multi-worker deployments (logs a startup WARNING).\n# backend: postgres -- Share dedupe state across pods via the application DB.\n# REQUIRED for any multi-replica deployment. If\n# database.backend is not 'postgres', a WARNING is logged and\n# the store falls back to the in-process memory store.\n#\n# dedupe_storage:\n# backend: auto\n\n # Recycle app ORM PostgreSQL connections before the environment's idle cutoff.\n pool_recycle: 300\n # App ORM PostgreSQL command timeout. Set to null to disable it or raise it\n # for intentionally long commands.\n command_timeout: 30\n # Restart required. Use one value across every process sharing this database.\n # full: current full-message checkpoints; delta: DeltaChannel for messages.\n checkpoint_channel_mode: full\n # Delta-mode tuning (only applies when checkpoint_channel_mode is delta).\n # Restart required, like the mode itself: the cadence is compiled into each\n # graph's channel table. All processes sharing this database must agree.\n # snapshot_frequency: full messages snapshot every N per-step writes\n # (higher = smaller checkpoints, slower materialization).\n checkpoint_delta:\n snapshot_frequency: 10\n # Size caps for the compiled checkpoint graph caches (per process).\n # Hot-reloadable, no restart needed: the value is re-read on each eviction\n # check and only changes when a cache evicts, never graph semantics.\n checkpoint_graph_cache:\n # Gateway thread-state accessor graphs (per assistant/mode/cadence).\n accessor_graph_max: 64\n\n# ============================================================================\n# Run Events Configuration\n# ============================================================================\n# Storage backend for run events (messages + execution traces).\n#\n# backend: memory -- No persistence, data lost on restart (default)\n# backend: db -- SQL database via ORM, full query capability (production)\n# backend: jsonl -- Append-only JSONL files (lightweight single-node persistence)\n#\n# run_events:\n# backend: memory\n# max_trace_content: 10240 # Truncation threshold for trace content (db backend, bytes)\n# track_token_usage: true # Accumulate token counts to RunRow\nrun_events:\n backend: memory\n max_trace_content: 10240\n track_token_usage: true\n\n# ============================================================================\n# Agent Storage Configuration\n# ============================================================================\n# Where custom agent DEFINITIONS (config.yaml + SOUL.md) are stored. This is\n# separate from `database` (run/thread/event data) and from agent memory.\n# Restart-required (the backend is captured at Gateway lifespan startup).\n#\n# backend: file -- Per-user files under {base_dir}/users/{uid}/agents/ (default).\n# Single-node only: an agent created on one node is invisible\n# to other nodes without a shared mount.\n# backend: db -- A row per agent in the shared SQL persistence layer, so every\n# node sees the same agents. Requires database.backend to be\n# 'sqlite' or 'postgres' (rejected at startup on 'memory').\n#\n# Switching an existing install to 'db'? Import the on-disk agents once with:\n# python backend/scripts/migrate_agents_to_db.py # --dry-run to preview\n# The source files are left in place, so reverting to 'file' is a clean rollback.\n#\n# agent_storage:\n# backend: file\nagent_storage:\n backend: file\n\n# ============================================================================\n# Scheduled Tasks Configuration\n# ============================================================================\n# Background scheduler for one-time and recurring (cron) agent runs.\n# All fields are restart-required (captured at Gateway lifespan startup).\n#\n# Multi-worker note: the scheduler runs once per uvicorn worker. SQLite silently\n# ignores row-level locks, so multiple workers can double-fire the same task.\n# For multi-worker deployments (GATEWAY_WORKERS > 1), use the Postgres database\n# backend, where FOR UPDATE SKIP LOCKED serializes claims correctly.\n#\n# scheduler:\n# enabled: false # Master switch for the background poller\n# poll_interval_seconds: 5 # How often to scan for due tasks\n# lease_seconds: 120 # Claim lease; a crashed process's task becomes reclaimable after this\n# max_concurrent_runs: 3 # Global cap on active scheduled runs; each poll claims only into the remaining budget\n# min_once_delay_seconds: 60 # Minimum future offset for one-time tasks at creation time\nscheduler:\n enabled: false\n poll_interval_seconds: 5\n lease_seconds: 120\n max_concurrent_runs: 3\n min_once_delay_seconds: 60\n\n# ============================================================================\n# Run Ownership Configuration\n# ============================================================================\n# Controls cross-process run ownership for multi-worker deployments.\n# When GATEWAY_WORKERS > 1, each worker claims runs with a lease; the heartbeat\n# renews leases, and reconciliation recovers orphaned runs from crashed workers.\n#\n# CLOCK-SYNC REQUIREMENT (multi-worker only): reconciliation compares another\n# worker's UTC lease timestamp against this worker's datetime.now(UTC). Worker\n# clocks MUST be synced (NTP / chrony / systemd-timesyncd — default on K8s and\n# cloud VMs) within a few seconds. grace_seconds is the skew budget; worst case\n# (owning worker's heartbeat just about to fire), a peer whose clock is more\n# than grace_seconds ahead can mis-reclaim a still-live run as an orphan. Raise\n# grace_seconds if your environment cannot keep clocks within a few seconds;\n# the trade-off is longer recovery latency for genuinely dead workers\n# (lease_seconds + grace_seconds from last heartbeat to reclaim).\n#\n# The owning worker itself does NOT use grace_seconds as extra execution time.\n# If renewal cannot be confirmed before lease_seconds expires, it cancels the\n# local run and suppresses durable finalization; grace_seconds only delays when\n# a peer may reclaim the expired row.\n\nrun_ownership:\n lease_seconds: 30 # Seconds before a run lease expires if not renewed.\n # Heartbeat renews every lease_seconds / 3.\n grace_seconds: 10 # Extra seconds past expiry before reclaiming an orphaned run.\n # Also the cross-worker clock-skew budget — see note above.\n heartbeat_enabled: false # Set to true for GATEWAY_WORKERS > 1\n\n# ============================================================================\n# Stream Bridge Configuration\n# ============================================================================\n# The stream bridge carries live agent events from gateway workers to SSE\n# clients. Docker Compose sets DEER_FLOW_STREAM_BRIDGE_REDIS_URL automatically,\n# so Docker deployments use Redis Streams even if this section is omitted.\n#\n# The redis bridge requires the optional `redis` extra. It is auto-detected from\n# this section on `make dev`, and always installed in the Docker image. To install\n# it manually: cd backend && uv sync --all-packages --extra redis\n#\n# stream_bridge:\n# type: memory # single-process only\n# queue_maxsize: 256 # data events retained per run. A reconnect\n# # older than this window receives SSE `gap`.\n#\n# stream_bridge:\n# type: redis # recommended for Docker / multi-worker gateway\n# redis_url: redis://redis:6379/0\n# queue_maxsize: 256 # data events retained per run (Redis MAXLEN).\n# # Older valid cursors receive SSE `gap`.\n# stream_ttl_seconds: 86400 # rolling TTL for retained stream buffers.\n# # Refreshed on each publish/publish_end; set 0\n# # to disable. This is not a run timeout.\n# recovered_stream_cleanup_delay_seconds: 60 # seconds to wait after\n# # publishing END for a recovered orphan run\n# # before deleting the stream key.\n# max_connections: 100 # optional pool ceiling. Each live SSE client\n# # holds one connection blocked in XREAD ... BLOCK\n# # for up to heartbeat_interval (15s), so hundreds\n# # of concurrent clients open hundreds of\n# # connections. Unset = redis-py default (unbounded).\n#\n# NOTE: the redis bridge is fail-hard in v1. Redis.from_url is lazy, so a down\n# Redis does not block gateway startup, but the first publish/xread raises. A\n# mid-run Redis outage fails the active run — there is no automatic retry/backoff\n# or fallback to the in-memory bridge. Run Redis with HA / a restart policy.\n\n# ============================================================================\n# User-Owned IM Channel Connections\n# ============================================================================\n# Lets logged-in users connect their own IM accounts from the DeerFlow frontend\n# while reusing the existing `channels` runtime configuration below.\n#\n# Security notes:\n# - No public IP, OAuth callback URL, or provider webhook is required.\n# - Provider bot/app credentials stay under `channels.*`.\n# - `channel_connections` stores per-user bindings and one-time connect codes.\n# - Telegram uses a deep link when `bot_username` is configured.\n# - Slack, Discord, Feishu, DingTalk, WeChat, and WeCom use `/connect `\n# through the already-running bot/app.\n#\n# channel_connections:\n# enabled: false\n# # Security: keep this enabled unless you intentionally want legacy open-bot behavior.\n# # Disabling it lets unbound external IM users create DeerFlow threads/runs.\n# require_bound_identity: true\n#\n# telegram:\n# enabled: false\n# bot_username: $TELEGRAM_BOT_USERNAME\n#\n# slack:\n# enabled: false\n#\n# discord:\n# enabled: false\n#\n# feishu:\n# enabled: false\n#\n# dingtalk:\n# enabled: false\n#\n# wechat:\n# enabled: false\n#\n# wecom:\n# enabled: false\n\n# ============================================================================\n# IM Channels Configuration\n# ============================================================================\n# Connect DeerFlow to external messaging platforms.\n# All channels use outbound connections (WebSocket or polling) — no public IP required.\n\n# channels:\n# # LangGraph-compatible Gateway API base URL for thread/message management (default: http://localhost:8001/api)\n# # For Docker deployments, use the Docker service name instead of localhost:\n# # langgraph_url: http://gateway:8001/api\n# # gateway_url: http://gateway:8001\n# langgraph_url: http://localhost:8001/api\n# # Gateway API URL for auxiliary queries like /models, /memory (default: http://localhost:8001)\n# gateway_url: http://localhost:8001\n# #\n# # Docker Compose note:\n# # If channels run inside the gateway container, use container DNS names instead\n# # of localhost, for example:\n# # langgraph_url: http://gateway:8001/api\n# # gateway_url: http://gateway:8001\n# # You can also set DEER_FLOW_CHANNELS_LANGGRAPH_URL / DEER_FLOW_CHANNELS_GATEWAY_URL.\n#\n# # Optional: default mobile/session settings for all IM channels\n# session:\n# assistant_id: lead_agent # or a custom agent name; custom agents route via lead_agent + agent_name\n# config:\n# recursion_limit: 100\n# context:\n# thinking_enabled: true\n# is_plan_mode: false\n# subagent_enabled: false\n#\n# feishu:\n# enabled: false\n# app_id: $FEISHU_APP_ID\n# app_secret: $FEISHU_APP_SECRET\n# # domain: https://open.feishu.cn # China (default)\n# # domain: https://open.larksuite.com # International\n#\n# slack:\n# enabled: false\n# bot_token: $SLACK_BOT_TOKEN # xoxb-...\n# app_token: $SLACK_APP_TOKEN # xapp-... (Socket Mode)\n# allowed_users: [] # empty = allow all; can also be a single Slack user ID string, e.g. U123456, but list form is recommended\n#\n# telegram:\n# enabled: false\n# bot_token: $TELEGRAM_BOT_TOKEN\n# allowed_users: [] # empty = allow all\n# rich_messages: false # Bot API 10.1 Rich Messages for final Markdown responses\n#\n# wechat:\n# enabled: false\n# bot_token: $WECHAT_BOT_TOKEN\n# ilink_bot_id: $WECHAT_ILINK_BOT_ID\n# # Optional: allow first-time QR bootstrap when bot_token is absent\n# qrcode_login_enabled: true\n# # Optional: sent as iLink-App-Id header when provided\n# ilink_app_id: \"\"\n# # Optional: sent as SKRouteTag header when provided\n# route_tag: \"\"\n# allowed_users: [] # empty = allow all\n# # Optional: timing values must be positive finite seconds\n# polling_timeout: 35\n# polling_retry_delay: 5\n# # QR poll interval when qrcode_login_enabled is true\n# qrcode_poll_interval: 2\n# # QR bootstrap timeout\n# qrcode_poll_timeout: 180\n# # Optional: persist getupdates cursor under the gateway container volume\n# state_dir: ./.deer-flow/wechat/state\n# # Optional: max inbound image size in bytes before skipping download\n# max_inbound_image_bytes: 20971520\n# # Optional: max outbound image size in bytes before skipping upload\n# max_outbound_image_bytes: 20971520\n# # Optional: max inbound file size in bytes before skipping download\n# max_inbound_file_bytes: 52428800\n# # Optional: max outbound file size in bytes before skipping upload\n# max_outbound_file_bytes: 52428800\n# # Optional: allowed file extensions for regular file receive/send\n# allowed_file_extensions: [\".txt\", \".md\", \".pdf\", \".csv\", \".json\", \".yaml\", \".yml\", \".xml\", \".html\", \".log\", \".zip\", \".doc\", \".docx\", \".xls\", \".xlsx\", \".ppt\", \".pptx\", \".rtf\"]\n#\n# # Optional: channel-level session overrides\n# session:\n# assistant_id: mobile-agent # custom agent names are supported here too\n# context:\n# thinking_enabled: false\n#\n# # Optional: per-user overrides by user_id\n# users:\n# \"123456789\":\n# assistant_id: vip-agent\n# config:\n# recursion_limit: 150\n# context:\n# thinking_enabled: true\n# subagent_enabled: true\n# wecom:\n# enabled: false\n# bot_id: $WECOM_BOT_ID\n# bot_secret: $WECOM_BOT_SECRET\n#\n# dingtalk:\n# enabled: false\n# client_id: $DINGTALK_CLIENT_ID\n# client_secret: $DINGTALK_CLIENT_SECRET\n# allowed_users: [] # empty = allow all\n# card_template_id: \"\" # Optional: AI Card template ID for streaming updates\n#\n# discord:\n# enabled: false\n# bot_token: $DISCORD_BOT_TOKEN\n# allowed_guilds: [] # empty = allow all guilds; can also be a single guild ID\n# mention_only: false # If true, only respond when the bot is mentioned\n# allowed_channels: [] # Optional: channel IDs exempt from mention_only (bot responds without mention)\n# thread_mode: false # If true, group a channel conversation into a thread\n\n# ============================================================================\n# Guardrails Configuration\n# ============================================================================\n# Optional pre-execution authorization for tool calls.\n# When enabled, every tool call passes through the configured provider\n# before execution. Three options: built-in allowlist, OAP policy provider,\n# or custom provider. See backend/docs/GUARDRAILS.md for full documentation.\n#\n# Providers are loaded by class path via resolve_variable (same as models/tools).\n\n# --- Option 1: Built-in AllowlistProvider (zero external deps) ---\n# guardrails:\n# enabled: true\n# provider:\n# use: deerflow.guardrails.builtin:AllowlistProvider\n# config:\n# denied_tools: [\"bash\", \"write_file\"]\n\n# --- Option 2: OAP passport provider (open standard, any implementation) ---\n# The Open Agent Passport (OAP) spec defines passport format and decision codes.\n# Any OAP-compliant provider works. Example using APort (reference implementation):\n# pip install aport-agent-guardrails && aport setup --framework deerflow\n# guardrails:\n# enabled: true\n# provider:\n# use: aport_guardrails.providers.generic:OAPGuardrailProvider\n\n# --- Option 3: Custom provider (any class with evaluate/aevaluate methods) ---\n# guardrails:\n# enabled: true\n# provider:\n# use: my_package:MyGuardrailProvider\n# config:\n# key: value\n\n# ============================================================================\n# Authorization Configuration\n# ============================================================================\n# Fine-grained resource authorization (RBAC and beyond). Disabled by default;\n# every authenticated user has access to all resources.\n# See RFC: https://github.com/bytedance/deer-flow/issues/4063\n#\n# authorization:\n# enabled: true\n# fail_closed: true # block on provider error / unresolved identity\n# default_role: user # applied when user_role is None; built-in RBAC requires this role below\n# provider:\n# use: deerflow.authz.rbac:RbacAuthorizationProvider\n# config:\n# # A known role with no policy for a resource is unrestricted for it.\n# # Define both `tools` and `routes` wherever access should be constrained.\n# roles:\n# admin:\n# tools: {allow: \"*\"}\n# routes: {allow: \"*\"}\n# models: {allow: \"*\"}\n# user:\n# tools: {allow: \"*\", deny: [\"update_agent\"]}\n# routes: {allow: \"*\"}\n# models: {allow: \"*\"}\n# guest:\n# tools: {allow: [\"web_search\", \"read_file\"]}\n# routes: {allow: [\"threads:read\", \"runs:read\"]}\n# models: {allow: [\"gpt-4o-mini\"]}\nauthorization:\n enabled: false\n\n# ============================================================================\n# Circuit Breaker Configuration\n# ============================================================================\n# Circuit breaker for LLM calls prevents repeated requests to a failing provider.\n# When the failure threshold is reached, subsequent calls fast-fail until recovery.\n#\n# This is useful for:\n# - Avoiding rate-limit bans during provider outages\n# - Reducing resource exhaustion from retry loops\n# - Gracefully degrading when LLM services are unavailable\n\n# circuit_breaker:\n# # Number of consecutive failures before opening the circuit (default: 5)\n# failure_threshold: 5\n# # Time in seconds before attempting to recover (default: 60)\n# recovery_timeout_sec: 60\n\n# ============================================================================\n# LLM Call Concurrency Configuration\n# ============================================================================\n# Cap the number of concurrently in-flight LLM calls process-wide. A provider\n# burst-rate (limit_burst_rate) error fires on the *slope* of the request rate,\n# not on a static quota - so the morning peak (e.g. 08:30) ramping from ~0 to\n# full throttle in seconds gets rejected even when total RPM is within budget.\n# Capping concurrency caps that slope. Retries alone make it worse (they add\n# demand inside the very burst being throttled); pair this cap with the\n# decorrelated-jitter backoff already built into the LLM error-handling\n# middleware, and ideally an nginx `limit_req` at the ingress.\n#\n# 0 disables the cap (default) - existing deployments see no behavior change.\n#\n# Per-process, not per-cluster: the cap bounds in-flight LLM calls within ONE\n# gateway process. With GATEWAY_WORKERS > 1 the aggregate cap across the\n# deployment is effectively `max_concurrent_calls * GATEWAY_WORKERS`, and a\n# multi-node rollout multiplies it further - size the per-process value with\n# that in mind (and pair it with an nginx `limit_req` at the ingress for a\n# true cluster-wide slope cap).\n#\n# Startup-only: the cap is captured at the first LLM run and frozen for the\n# process lifetime. Editing `max_concurrent_calls` here takes effect only after\n# a gateway restart; the other `llm_call.*` knobs below remain hot-reloadable.\n# Freezing the cap avoids the downscale and config-freshness races that a\n# runtime-mutable, process-wide/cross-loop limiter would otherwise hit (a\n# lowered cap could keep admitting queued waiters, or a stale config snapshot\n# constructed after a fresher one could restore a higher cap).\n\n# llm_call:\n# # Max concurrently in-flight LLM calls across the whole process (default: 0 = disabled)\n# max_concurrent_calls: 0\n# # Max LLM call attempts for retriable transient errors, 1 = no retry (default: 3)\n# retry_max_attempts: 3\n# # Base delay (ms) for the decorrelated-jitter retry backoff (default: 1000)\n# retry_base_delay_ms: 1000\n# # Hard cap (ms) on any single retry backoff delay (default: 8000)\n# retry_cap_delay_ms: 8000\n# # Backoff base (ms) used ONLY for burst-rate (limit_burst_rate) 429s - higher\n# # than retry_base_delay_ms so the single burst retry lands after the throttle\n# # window subsides. Ignored when the provider sends Retry-After (default: 5000)\n# burst_retry_base_delay_ms: 5000\n\n# ============================================================================\n# SSO / OIDC Authentication (optional)\n# ============================================================================\n# Enable SSO login via any OIDC-compatible provider (Keycloak, Google, Azure AD, Okta, etc.).\n# When enabled, the login page will show SSO buttons alongside the standard email/password form.\n#\n# Provider configuration:\n# - issuer: The OIDC issuer URL (e.g. https://keycloak.example.com/realms/deerflow)\n# - client_id: OAuth2 client ID from the provider\n# - client_secret: OAuth2 client secret ($ENV_VAR references supported)\n# - redirect_uri: Callback URL. In production, this must match what you configure in\n# the provider. Defaults to a self-derived URL in development.\n#\n# Keycloak setup:\n# 1. Create a client with type \"confidential\" and Standard Flow enabled\n# 2. Add Valid Redirect URI: http://localhost:8001/api/v1/auth/callback/keycloak\n# 3. Add Web Origin: http://localhost:8001 (or your frontend origin)\n\n# ============================================================================\n# Local (email/password) authentication\n# ============================================================================\n# Self-registration via POST /api/v1/auth/register is open by default: anyone\n# who can reach the Gateway may create a regular user account.\n#\n# The OIDC provisioning policy below (allowed_email_domains, require_verified_email,\n# auto_create_users) is enforced only in the SSO callback. It does NOT constrain local\n# registration, so on an SSO-provisioned deployment leaving this open lets a visitor\n# create an account outside that policy. Set allow_registration: false there.\n#\n# The first admin account is always created through /initialize regardless of this\n# setting, so turning it off cannot lock you out of a fresh install.\n\n# auth:\n# local:\n# allow_registration: false\n#\n# oidc:\n# enabled: true\n# # Base URL of the frontend, used for redirects after SSO callback.\n# # In production behind a reverse proxy, set this to the public frontend URL\n# # (e.g. https://deerflow.example.com). In development, leave unset.\n# # frontend_base_url: http://localhost:3000\n# providers:\n# keycloak:\n# display_name: Keycloak\n# issuer: https://keycloak.example.com/realms/deerflow\n# client_id: deerflow\n# client_secret: $KEYCLOAK_CLIENT_SECRET\n# # Optional: explicitly set the callback URL.\n# # redirect_uri: https://deerflow.example.com/api/v1/auth/callback/keycloak\n# scopes:\n# - openid\n# - email\n# - profile\n# token_endpoint_auth_method: client_secret_post\n#\n# # User provisioning settings (safe defaults shown below):\n# auto_create_users: true # Auto-create DeerFlow account on first SSO login\n# require_verified_email: true # Reject SSO logins without verified email\n# # allowed_email_domains: # Restrict to specific email domains\n# # - example.com\n# admin_emails: [] # Auto-grant admin role to these emails\n#\n# # Security features (enabled by default):\n# pkce_enabled: true # PKCE (S256) for authorization code flow\n# nonce_enabled: true # Nonce validation in ID tokens\n"}, "bec6277930d6ee73c58156689f1556780724e35d": {"t": 1785846908.91028, "n": 120226, "h": "4cc9f0bccfea2fcc", "c": "# Configuration for the DeerFlow application\n#\n# Guidelines:\n# - Copy this file to `config.yaml` and customize it for your environment\n# - The default path of this configuration file is `config.yaml` in the project root.\n# You can set `DEER_FLOW_PROJECT_ROOT` to define that root explicitly, or use\n# `DEER_FLOW_CONFIG_PATH` to point at a specific config file.\n# - Runtime state defaults to `.deer-flow` under the project root. Override it\n# with `DEER_FLOW_HOME` when you need a different writable data directory.\n# - Environment variables are available for all field values. Example: `api_key: $OPENAI_API_KEY`\n# - The `use` path is a string that looks like \"package_name.sub_package_name.module_name:class_name/variable_name\".\n\n# ============================================================================\n# Config Version (used to detect outdated config files)\n# ============================================================================\n# Bump this number when the config schema changes.\n# Run `make config-upgrade` to merge new fields into your local config.yaml.\nconfig_version: 32\n\n# ============================================================================\n# Logging\n# ============================================================================\n# Log level for deerflow modules (debug/info/warning/error)\nlog_level: info\n\n# Request trace correlation for Gateway logs, HTTP response headers, and\n# Langfuse metadata. Disabled by default to preserve existing HTTP/log output.\nlogging:\n enhance:\n enabled: false\n format: text\n\n# ============================================================================\n# Agent Extensions\n# ============================================================================\n# Optional AgentMiddleware classes loaded into the lead and subagent runtime\n# middleware chains after built-in runtime middlewares, but before the\n# safety/clarification tail. Missing packages, invalid classes, and broken\n# modules fail loudly at agent creation with an actionable import error.\n# The same zero-argument class list applies to both lead and subagent runtimes;\n# lead-only vs subagent-only configuration is not expressible yet. Treat these\n# files as trusted operator config because middleware classes execute code.\n# Uncomment this block to define middlewares in config.yaml. Leaving it commented\n# lets extensions_config.json remain the source of truth for extension packages.\n# extensions:\n# middlewares:\n# - my_company.deerflow_middlewares:DomainGuardMiddleware\n# - my_company.deerflow_middlewares:LatencyStampingMiddleware\n\n# ============================================================================\n# Tracing / Observability (Monocle)\n# ============================================================================\n# Optional agent tracing via Monocle. Configured through environment variables\n# (MONOCLE_TRACING, MONOCLE_EXPORTERS, OKAHU_API_KEY — like LangSmith/Langfuse),\n# not config.yaml keys, and OFF by default. See README.md → \"Monocle Tracing\"\n# for setup, what each exporter captures, and where the trace data goes.\n\n# ============================================================================\n# Token Usage\n# ============================================================================\n# Enable token usage collection and display.\n# When enabled, DeerFlow records input/output/total tokens per model call\n# and shows usage metadata in the workspace UI when providers return it.\ntoken_usage:\n enabled: true\n\n# ============================================================================\n# Token Budget — Per-run token limits\n# ============================================================================\n# Prevents runaway API costs by enforcing hard token limits per run.\n# When warn_threshold is crossed, the agent receives an in-context warning.\n# When hard_stop_threshold is crossed, tool_calls are stripped and the agent\n# is forced to produce a final answer immediately.\ntoken_budget:\n enabled: false # Set to true to activate budget enforcement\n max_tokens: 200000 # Total token limit (input + output) per run\n max_input_tokens: null # Optional separate input-only limit\n max_output_tokens: null # Optional separate output-only limit\n warn_threshold: 0.8 # Warn at 80% of the budget\n hard_stop_threshold: 1.0 # Force stop at 100% of the budget\n\n# ============================================================================\n# Recursion Limit — Hard ceiling for a client-supplied run recursion_limit\n# ============================================================================\n# A run's recursion_limit caps the number of LangGraph super-steps (each is at\n# least one LLM call). The Gateway never trusts a client-supplied value\n# verbatim: any value above this ceiling is clamped down to it, preventing\n# runaway API cost / DoS. Invalid or non-positive client values fall back to\n# the server default of 100. Raise this only if you legitimately run very\n# deeply nested subagent graphs.\nmax_recursion_limit: 1000\n\n\n# ============================================================================\n# Models Configuration\n# ============================================================================\n# Configure available LLM models for the agent to use.\n#\n# Two token fields look similar but mean different things:\n# - `max_tokens` is the per-call OUTPUT cap passed to the provider.\n# - `context_window` is a positive integer for the total context capacity\n# (prompt + completion) and drives the real-time \"% context used\" indicator\n# in the chat UI.\n# Leave `context_window` unset if the provider limit is unknown; the percentage\n# will not render. Verify configured values against the provider's model docs.\n#\n# Optional per-model pricing (powers the real-cost display on the workspace\n# console). Add a `pricing` block to any model entry; use ONE currency across\n# all models. Mixed currencies disable cost reporting to prevent invalid sums.\n# Prices are per one million tokens.\n#\n# pricing:\n# currency: CNY # ISO code shown in the console (CNY, USD, ...)\n# input_per_million: 8.0 # price per 1M input tokens (cache miss)\n# output_per_million: 32.0 # price per 1M output tokens\n# input_cache_hit_per_million: 0.8 # price per 1M cache-hit input tokens\n# # (optional; omit → hits billed at miss price)\n\nmodels:\n # Example: Volcengine (Doubao) model\n # - name: doubao-seed-1.8\n # display_name: Doubao-Seed-1.8\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: doubao-seed-1-8-251228\n # api_base: https://ark.cn-beijing.volces.com/api/v3\n # api_key: $VOLCENGINE_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # context_window: 262144 # Total prompt + completion capacity\n # supports_thinking: true\n # supports_vision: true\n # supports_reasoning_effort: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Volcengine Coding Plan (one key, multi-vendor gateway)\n # The Coding Plan endpoint (/api/coding/v3) lets you access models from\n # Doubao, GLM, DeepSeek, Kimi, and MiniMax with a single API key.\n # Each model may differ in thinking/vision support - configure per-model.\n #\n # - name: glm-5.2-cp\n # display_name: GLM-5.2 (Coding Plan)\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: glm-5.2\n # api_base: https://ark.cn-beijing.volces.com/api/coding/v3\n # api_key: $VOLCENGINE_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # supports_thinking: true\n # supports_vision: false\n # supports_reasoning_effort: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n #\n # - name: deepseek-v4-pro-cp\n # display_name: DeepSeek-V4-Pro (Coding Plan)\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: deepseek-v4-pro\n # api_base: https://ark.cn-beijing.volces.com/api/coding/v3\n # api_key: $VOLCENGINE_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # supports_thinking: true\n # supports_vision: false\n # supports_reasoning_effort: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n # Example: OpenAI model\n # - name: gpt-4\n # display_name: GPT-4\n # use: langchain_openai:ChatOpenAI\n # model: gpt-4\n # api_key: $OPENAI_API_KEY # Use environment variable\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096 # Per-call output cap\n # context_window: 128000 # Total prompt + completion capacity\n # temperature: 0.7\n # supports_vision: true # Enable vision support for view_image tool\n\n # Example: OpenAI Responses API model\n # - name: gpt-5-responses\n # display_name: GPT-5 (Responses API)\n # use: langchain_openai:ChatOpenAI\n # model: gpt-5\n # api_key: $OPENAI_API_KEY\n # request_timeout: 600.0\n # max_retries: 2\n # use_responses_api: true\n # output_version: responses/v1\n # context_window: 400000\n # supports_vision: true\n\n # Example: Ollama (native provider — preserves thinking/reasoning content)\n #\n # IMPORTANT: Use langchain_ollama:ChatOllama instead of langchain_openai:ChatOpenAI\n # for Ollama models. The OpenAI-compatible endpoint (/v1/chat/completions) does NOT\n # return reasoning_content as a separate field — thinking content is either flattened\n # into tags or dropped entirely (ollama/ollama#15293). The native Ollama API\n # (/api/chat) correctly separates thinking from response content.\n #\n # Install: cd backend && uv pip install 'deerflow-harness[ollama]'\n #\n # - name: qwen3-local\n # display_name: Qwen3 32B (Ollama)\n # use: langchain_ollama:ChatOllama\n # model: qwen3:32b\n # base_url: http://localhost:11434 # No /v1 suffix — uses native /api/chat\n # num_predict: 8192\n # temperature: 0.7\n # reasoning: true # Passes think:true to Ollama native API\n # context_window: 32768 # Match the context length served by Ollama\n # supports_thinking: true\n # supports_vision: false\n #\n # - name: gemma4-local\n # display_name: Gemma 4 27B (Ollama)\n # use: langchain_ollama:ChatOllama\n # model: gemma4:27b\n # base_url: http://localhost:11434\n # num_predict: 8192\n # temperature: 0.7\n # reasoning: true\n # context_window: 32768\n # supports_thinking: true\n # supports_vision: true\n #\n # For Docker deployments, use host.docker.internal instead of localhost:\n # base_url: http://host.docker.internal:11434\n\n # Example: Anthropic Claude model (with extended thinking)\n # supports_thinking: true is required — without it, DeerFlow silently falls\n # back to non-thinking mode even when the UI thinking toggle is on.\n # budget_tokens is required by the Anthropic API when thinking.type=enabled\n # (no server default; min 1024; must be less than max_tokens).\n # - name: claude-sonnet-4\n # display_name: Claude Sonnet 4\n # use: langchain_anthropic:ChatAnthropic\n # model: claude-sonnet-4-20250514\n # api_key: $ANTHROPIC_API_KEY\n # default_request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 16000 # Per-call output cap\n # context_window: 200000 # Total prompt + completion capacity\n # supports_vision: true\n # supports_thinking: true\n # when_thinking_enabled:\n # thinking:\n # type: enabled\n # budget_tokens: 4096 # required; min 1024; must be < max_tokens\n # when_thinking_disabled:\n # thinking:\n # type: disabled\n\n # Example: Google Gemini model (native SDK, no thinking support)\n # - name: gemini-2.5-pro\n # display_name: Gemini 2.5 Pro\n # use: langchain_google_genai:ChatGoogleGenerativeAI\n # model: gemini-2.5-pro\n # gemini_api_key: $GEMINI_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # context_window: 1048576\n # supports_vision: true\n\n # Example: Gemini model via OpenAI-compatible gateway (with thinking support)\n # Use PatchedChatOpenAI so that tool-call thought_signature values on tool_calls\n # are preserved across multi-turn tool-call conversations — required by the\n # Gemini API when thinking is enabled. See:\n # https://docs.cloud.google.com/vertex-ai/generative-ai/docs/thought-signatures\n # - name: gemini-2.5-pro-thinking\n # display_name: Gemini 2.5 Pro (Thinking)\n # use: deerflow.models.patched_openai:PatchedChatOpenAI\n # model: google/gemini-2.5-pro-preview # model name as expected by your gateway\n # api_key: $GEMINI_API_KEY\n # base_url: https:///v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 16384\n # context_window: 1048576\n # supports_thinking: true\n # supports_vision: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Xiaomi MiMo model (with thinking support)\n # MiMo thinking mode returns reasoning_content and requires that field to be\n # replayed on historical assistant messages in multi-turn agent/tool-call\n # conversations. Use PatchedChatMiMo instead of plain ChatOpenAI.\n # Use https://api.xiaomimimo.com/v1 with pay-as-you-go `sk-...` keys.\n # Use your Token Plan regional URL (for example\n # https://token-plan-cn.xiaomimimo.com/v1) with Token Plan `tp-...` keys.\n # PatchedChatMiMo is model-id agnostic; use it for every MiMo thinking model\n # entry you configure (for example mimo-v2.5-pro, mimo-v2.5, mimo-v2-pro,\n # mimo-v2-omni, or mimo-v2-flash), including models referenced by subagent\n # model overrides.\n # See: https://platform.xiaomimimo.com/docs/en-US/usage-guide/passing-back-reasoning_content\n # - name: mimo-v2.5-pro\n # display_name: MiMo V2.5 Pro\n # use: deerflow.models.patched_mimo:PatchedChatMiMo\n # model: mimo-v2.5-pro\n # api_key: $MIMO_API_KEY\n # base_url: https://api.xiaomimimo.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: DeepSeek V4 model (with thinking support)\n # - name: deepseek-v4\n # display_name: DeepSeek V4 (Thinking)\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: deepseek-v4-pro\n # api_key: $DEEPSEEK_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false # DeepSeek V4 does not support vision\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Kimi K2.5 model\n # - name: kimi-k2.5\n # display_name: Kimi K2.5\n # use: deerflow.models.patched_deepseek:PatchedChatDeepSeek\n # model: kimi-k2.5\n # api_base: https://api.moonshot.cn/v1\n # api_key: $MOONSHOT_API_KEY\n # timeout: 600.0\n # max_retries: 2\n # max_tokens: 32768\n # supports_thinking: true\n # supports_vision: true # Check your specific model's capabilities\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: Novita AI (OpenAI-compatible)\n # Novita provides an OpenAI-compatible API with competitive pricing\n # See: https://novita.ai\n # - name: novita-deepseek-v3.2\n # display_name: Novita DeepSeek V3.2\n # use: langchain_openai:ChatOpenAI\n # model: deepseek/deepseek-v3.2\n # api_key: $NOVITA_API_KEY\n # base_url: https://api.novita.ai/openai\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 0.7\n # supports_thinking: true\n # supports_vision: true\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: StepFun (阶跃星辰) reasoning models\n # StepFun provides OpenAI-compatible API with reasoning models.\n # With reasoning_format: deepseek-style, the API returns reasoning_content\n # (same field as DeepSeek), which must be replayed on historical assistant\n # messages in multi-turn tool-call conversations.\n # Use PatchedChatStepFun instead of plain ChatOpenAI.\n # Docs: https://platform.stepfun.com/docs/api-reference/chat-completions\n # - name: step-3.7-flash\n # display_name: Step 3.7 Flash\n # use: deerflow.models.patched_stepfun:PatchedChatStepFun\n # model: step-3.7-flash\n # api_key: $STEPFUN_API_KEY\n # base_url: https://api.stepfun.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # supports_thinking: true\n # supports_reasoning_effort: true\n # supports_vision: true\n # when_thinking_enabled:\n # extra_body:\n # reasoning_format: deepseek-style\n # when_thinking_disabled:\n # extra_body:\n # reasoning_format: deepseek-style\n\n # Example: MiniMax (OpenAI-compatible) - International Edition\n # MiniMax provides high-performance models with 512K context window and 128K max output\n # Docs: https://platform.minimax.io/docs/api-reference/text-openai-api\n # - name: minimax-m3\n # display_name: MiniMax M3\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M3\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimax.io/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: true\n # supports_thinking: true\n # # PatchedChatMiniMax is the MiniMax adapter: it enables reasoning_split and\n # # maps MiniMax's structured reasoning into reasoning_content (the field\n # # DeerFlow understands), and it strips the per-message `name` field that\n # # DeerFlow middlewares attach — MiniMax rejects requests whose user-message\n # # names differ with \"user name must be consistent (2013)\". Declare the\n # # thinking toggle so non-thinking paths (flash mode, follow-up suggestions,\n # # title/memory generation) truly disable reasoning instead of spending\n # # tokens on it.\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: adaptive\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # NOTE: M2.x models always think — passing thinking:{type:disabled} has no\n # effect (per MiniMax docs), so the toggle above is omitted for M2.7. The\n # follow-up-suggestions endpoint strips inline defensively regardless.\n # Still use the PatchedChatMiniMax adapter: it strips the per-message `name`\n # field DeerFlow middlewares attach, which MiniMax otherwise rejects with\n # \"user name must be consistent (2013)\".\n # - name: minimax-m2.7\n # display_name: MiniMax M2.7\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimax.io/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # - name: minimax-m2.7-highspeed\n # display_name: MiniMax M2.7 Highspeed\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7-highspeed\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimax.io/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # Example: MiniMax (OpenAI-compatible) - CN 中国区用户\n # MiniMax provides high-performance models with 512K context window and 128K max output\n # Docs: https://platform.minimaxi.com/docs/api-reference/text-openai-api\n # - name: minimax-m3\n # display_name: MiniMax M3\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M3\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimaxi.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: true\n # supports_thinking: true\n # # PatchedChatMiniMax is the MiniMax adapter: it enables reasoning_split and\n # # maps MiniMax's structured reasoning into reasoning_content (the field\n # # DeerFlow understands), and it strips the per-message `name` field that\n # # DeerFlow middlewares attach — MiniMax rejects requests whose user-message\n # # names differ with \"user name must be consistent (2013)\". Declare the\n # # thinking toggle so non-thinking paths (flash mode, follow-up suggestions,\n # # title/memory generation) truly disable reasoning instead of spending\n # # tokens on it.\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: adaptive\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # NOTE: M2.x models always think — passing thinking:{type:disabled} has no\n # effect (per MiniMax docs), so the toggle above is omitted for M2.7. The\n # follow-up-suggestions endpoint strips inline defensively regardless.\n # Still use the PatchedChatMiniMax adapter: it strips the per-message `name`\n # field DeerFlow middlewares attach, which MiniMax otherwise rejects with\n # \"user name must be consistent (2013)\".\n # - name: minimax-m2.7\n # display_name: MiniMax M2.7\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimaxi.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # - name: minimax-m2.7-highspeed\n # display_name: MiniMax M2.7 Highspeed\n # use: deerflow.models.patched_minimax:PatchedChatMiniMax\n # model: MiniMax-M2.7-highspeed\n # api_key: $MINIMAX_API_KEY\n # base_url: https://api.minimaxi.com/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 4096\n # temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0]\n # supports_vision: false # M2.7 is text-only; M3 supports vision\n # supports_thinking: true\n\n # Example: OpenRouter (OpenAI-compatible)\n # OpenRouter models use the same ChatOpenAI + base_url pattern as other OpenAI-compatible gateways.\n # - name: openrouter-gemini-2.5-flash\n # display_name: Gemini 2.5 Flash (OpenRouter)\n # use: langchain_openai:ChatOpenAI\n # model: google/gemini-2.5-flash-preview\n # api_key: $OPENAI_API_KEY\n # base_url: https://openrouter.ai/api/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # temperature: 0.7\n\n # Example: Atlas Cloud (OpenAI-compatible)\n # Atlas Cloud exposes a single OpenAI-compatible endpoint in front of many open\n # models (DeepSeek, Qwen, Kimi, GLM, MiniMax, Llama, ...), so it uses the same\n # ChatOpenAI + base_url pattern as other OpenAI-compatible gateways.\n # Browse model ids at https://api.atlascloud.ai/v1/models — see https://atlascloud.ai\n # - name: atlascloud-deepseek-v3.2\n # display_name: DeepSeek V3.2 (Atlas Cloud)\n # use: langchain_openai:ChatOpenAI\n # model: deepseek-ai/DeepSeek-V3.2-Exp\n # api_key: $ATLASCLOUD_API_KEY\n # base_url: https://api.atlascloud.ai/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # temperature: 0.7\n # supports_vision: false\n #\n # For reasoning models on Atlas Cloud (e.g. a Qwen3 *-thinking id), use the\n # patched OpenAI-compatible adapter so reasoning_content is replayed across\n # multi-turn tool-call conversations:\n # - name: atlascloud-qwen3-thinking\n # display_name: Qwen3 235B Thinking (Atlas Cloud)\n # use: deerflow.models.patched_openai:PatchedChatOpenAI\n # model: qwen/qwen3-235b-a22b-thinking-2507\n # api_key: $ATLASCLOUD_API_KEY\n # base_url: https://api.atlascloud.ai/v1\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false\n # when_thinking_enabled:\n # extra_body:\n # thinking:\n # type: enabled\n # when_thinking_disabled:\n # extra_body:\n # thinking:\n # type: disabled\n\n # Example: vLLM 0.19.0 (OpenAI-compatible, with reasoning toggle)\n # DeerFlow's vLLM provider preserves vLLM reasoning across tool-call turns and\n # toggles Qwen-style reasoning by writing\n # extra_body.chat_template_kwargs.enable_thinking=true/false.\n # Some reasoning models also require the server to be started with\n # `vllm serve ... --reasoning-parser `.\n # - name: qwen3-32b-vllm\n # display_name: Qwen3 32B (vLLM)\n # use: deerflow.models.vllm_provider:VllmChatModel\n # model: Qwen/Qwen3-32B\n # api_key: $VLLM_API_KEY\n # base_url: http://localhost:8000/v1\n # # Enable only when the endpoint reports cumulative usage on every stream chunk.\n # cumulative_stream_usage: true\n # request_timeout: 600.0\n # max_retries: 2\n # max_tokens: 8192\n # supports_thinking: true\n # supports_vision: false\n # when_thinking_enabled:\n # extra_body:\n # chat_template_kwargs:\n # enable_thinking: true\n\n\n # Example: Qwen3-Coder deployed on MindIE Engine\n # - name: Qwen3_Coder_480B_MindIE\n # display_name: Qwen3-Coder-480B (MindIE)\n # use: deerflow.models.mindie_provider:MindIEChatModel\n # model: Qwen3-Coder-480B-A35B-Instruct-Client\n # base_url: http://localhost:8989/v1\n # api_key: $OPENAI_API_KEY\n # temperature: 0\n # max_retries: 1\n # supports_thinking: false\n # supports_vision: false\n # supports_reasoning_effort: false\n # # --- Advanced Network Settings ---\n # # Due to MindIE's streaming limitations with tool calling, the provider\n # # uses mock-streaming (awaiting full generation). Extended timeouts are required.\n # read_timeout: 900.0 # 15 minutes to prevent drops during long document generation\n # connect_timeout: 30.0\n # write_timeout: 60.0\n # pool_timeout: 30.0\n\n# ============================================================================\n# Tool Groups Configuration\n# ============================================================================\n# Define groups of tools for organization and access control\n\ntool_groups:\n - name: web\n - name: file:read\n - name: file:write\n - name: bash\n - name: browser\n\n# ============================================================================\n# Tools Configuration\n# ============================================================================\n# Configure available tools for the agent to use\n\ntools:\n # Web search tool (uses DuckDuckGo, no API key required)\n - name: web_search\n group: web\n use: deerflow.community.ddg_search.tools:web_search_tool\n max_results: 5\n # backend: auto # DDGS backend(s): auto, duckduckgo, brave, wikipedia, etc.\n # region: wt-wt # wt-wt is normalized for Wikipedia when backend includes auto/all/wikipedia.\n # safesearch: moderate # on, moderate, off\n\n # Web search tool (uses SearXNG, self-hosted, no API key required)\n # SearXNG is a free internet metasearch engine which aggregates results from\n # various search services. Deploy your own instance: https://github.com/searxng/searxng\n # For Docker deployments, use the Docker service name instead of localhost.\n # - name: web_search\n # group: web\n # use: deerflow.community.searxng.tools:web_search_tool\n # base_url: http://localhost:8088 # SearXNG instance URL (default: :8088; Docker: http://searxng:8080)\n # max_results: 5 # Maximum number of search results\n\n # Web search tool (uses Serper - Google Search API, requires SERPER_API_KEY)\n # Serper provides real-time Google Search results. Sign up at https://serper.dev\n # Note: set SERPER_API_KEY in your environment before starting the app.\n # Avoid putting literal API keys in config.yaml; use the $VAR form instead.\n # - name: web_search\n # group: web\n # use: deerflow.community.serper.tools:web_search_tool\n # max_results: 5 # capped at 10 by the Serper provider\n # # api_key: $SERPER_API_KEY # Optional explicit env-var reference\n\n # Web search tool (uses Brave Search API, requires BRAVE_SEARCH_API_KEY)\n # Brave Search returns results from an independent index. Sign up at\n # https://brave.com/search/api/ to get a key. Unlike the DuckDuckGo\n # `backend: brave` option above, this calls the official Brave API directly.\n # - name: web_search\n # group: web\n # use: deerflow.community.brave.tools:web_search_tool\n # max_results: 5 # Capped at 20 by the Brave Search API\n # # api_key: $BRAVE_SEARCH_API_KEY # Optional if the env var is set\n\n # Web search tool (requires Tavily API key)\n # - name: web_search\n # group: web\n # use: deerflow.community.tavily.tools:web_search_tool\n # max_results: 5\n # # api_key: $TAVILY_API_KEY # Set if needed\n\n # Web search tool (uses InfoQuest, requires InfoQuest API key)\n # - name: web_search\n # group: web\n # use: deerflow.community.infoquest.tools:web_search_tool\n # # Used to limit the scope of search results, only returns content within the specified time range. Set to -1 to disable time filtering\n # search_time_range: 10\n\n # Web search tool (uses Exa, requires EXA_API_KEY)\n # - name: web_search\n # group: web\n # use: deerflow.community.exa.tools:web_search_tool\n # max_results: 5\n # search_type: auto # Options: auto, neural, keyword\n # contents_max_characters: 1000\n # # api_key: $EXA_API_KEY\n\n # Web search tool (uses Firecrawl, requires FIRECRAWL_API_KEY)\n # - name: web_search\n # group: web\n # use: deerflow.community.firecrawl.tools:web_search_tool\n # max_results: 5\n # # api_key: $FIRECRAWL_API_KEY\n\n # Web search tool (uses GroundRoute, requires GROUNDROUTE_API_KEY)\n # GroundRoute is a meta search layer: one API in front of six engines (Serper,\n # Brave, Exa, Tavily, Firecrawl, Perplexity). It routes each query to the cheapest\n # engine that clears a quality bar and fails over if one is down. Pricing is\n # gain-share (you keep about half of any cache savings). Get a key at\n # https://groundroute.ai/keys\n # - name: web_search\n # group: web\n # use: deerflow.community.groundroute.tools:web_search_tool\n # max_results: 5 # Clamped to 1-50 by GroundRoute\n # # api_key: $GROUNDROUTE_API_KEY # Optional if the env var is set\n\n # Web search tool (uses fastCRW - Firecrawl-compatible web scraper, single binary,\n # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.)\n # - name: web_search\n # group: web\n # use: deerflow.community.fastcrw.tools:web_search_tool\n # max_results: 5\n # # api_key: $CRW_API_KEY\n # # base_url: https://fastcrw.com/api # default cloud; set to e.g. http://localhost:3000 for self-host\n\n # Web fetch tool (uses Browserless - headless Chrome, self-hosted or cloud)\n # Browserless renders pages with a real headless Chrome, ideal for JavaScript-heavy\n # sites and SPAs. Deploy your own: https://github.com/browserless/browserless\n # For Docker deployments, use the Docker service name instead of localhost.\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry below before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.browserless.tools:web_fetch_tool\n # base_url: http://localhost:3032 # Browserless instance URL (default: :3032; Docker: http://browserless:3000)\n # # token: $BROWSERLESS_TOKEN # API token (required for Browserless Cloud; optional for self-hosted)\n # timeout_s: 30 # Request timeout in seconds\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets.\n # # wait_for_event: \"networkidle\" # Wait for a page event before returning (e.g. \"load\", \"networkidle\")\n # # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds\n # # wait_for_selector: \"article\" # CSS selector to wait for before returning\n\n # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no third-party API key)\n # Crawl4AI returns server-cleaned \"fit\" markdown directly (no readability step needed),\n # ideal for JavaScript-heavy sites. Self-host it:\n # docker run -d -p 11235:11235 --shm-size=1g \\\n # -e CRAWL4AI_API_TOKEN=$CRAWL4AI_TOKEN unclecode/crawl4ai:0.9.2\n # Crawl4AI >= 0.9 is secure-by-default: a bearer token is REQUIRED on every request\n # except GET /health, and a server started without CRAWL4AI_API_TOKEN binds 127.0.0.1\n # only (so a container/remote DeerFlow cannot reach it at all). Set the same value in\n # the server env and in `token:` below, or requests fail with HTTP 401.\n # Use >= 0.8.7: earlier images, including 0.8.6, carry known pre-auth RCEs.\n # For Docker deployments, use the Docker service name instead of localhost.\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry below before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.crawl4ai.tools:web_fetch_tool\n # base_url: http://localhost:11235 # Crawl4AI server URL (Docker: http://crawl4ai:11235)\n # token: $CRAWL4AI_TOKEN # Bearer token; required by Crawl4AI >= 0.9\n # timeout: 30 # Request timeout in seconds\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets.\n # # filter: fit # Markdown filter: fit (default) | raw | bm25 | llm\n\n # Web capture tool (uses Browserless /screenshot to render a page as an artifact)\n # Browserless captures JavaScript-heavy pages with a real headless Chrome and\n # writes the screenshot into the current thread's outputs. It can run against\n # a self-hosted Browserless instance without a token, or Browserless Cloud with\n # BROWSERLESS_TOKEN. For Docker deployments, use the Docker service name instead\n # of localhost.\n # - name: web_capture\n # group: web\n # use: deerflow.community.browserless.tools:web_capture_tool\n # base_url: http://localhost:3032 # Browserless instance URL (Docker: http://browserless:3000)\n # # token: $BROWSERLESS_TOKEN # Required for Browserless Cloud; optional for self-hosted\n # timeout_s: 30 # Request timeout in seconds (`timeout` also accepted)\n # output_format: png # png, jpeg, or webp\n # full_page: true # Capture entire page instead of viewport only\n # viewport_width: 1280\n # viewport_height: 720\n # # wait_for_selector: \"main\" # CSS selector to wait for before capturing\n # # wait_for_selector_timeout_ms: 5000\n # # wait_for_timeout_ms: 1000 # Extra wait after navigation in milliseconds\n # # best_attempt: true # Continue with current page state if waits time out\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY to\n # # # capture internal/private targets (loopback, RFC1918, etc.)\n\n # Agentic browser control (stateful navigate → observe → click/type loop).\n # Unlike web_fetch/web_capture (read-only, stateless), these tools keep a live\n # per-thread Playwright browser so the agent can interact with JavaScript-heavy\n # pages, fill forms, and follow multi-step flows. Elements are addressed by the\n # numeric [ref] index returned in each page snapshot.\n #\n # Requires the optional dependency, installed once per environment:\n # cd backend && uv sync --extra browser && uv run playwright install chromium\n # make dev / Docker startup auto-detect an active browser_navigate entry and\n # preserve the browser extra during dependency sync.\n #\n # Uncomment the whole block to enable. Launch and URL-policy settings are read\n # canonically from browser_navigate and shared by every browser tool + Live.\n # - name: browser_navigate\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_navigate_tool\n # headless: true # Set false only for local, trusted debugging\n # timeout_ms: 30000 # Per-action navigation/interaction timeout\n # viewport_width: 1280\n # viewport_height: 720\n # # allow_private_addresses: false # SSRF guard: keep false in production\n # # cdp_url: http://127.0.0.1:9222 # Attach to YOUR running Chrome (Codex-style)\n # # allow_unguarded_cdp: false # REQUIRED true with cdp_url: attaching to an\n # # # existing Chrome prevents DeerFlow from\n # # # enforcing its subresource/redirect SSRF guard.\n # # # instead of launching a private headless one.\n # # # Start Chrome with --remote-debugging-port=9222\n # # # so you watch the agent drive your real browser\n # # # with your real login sessions. Local/trusted only;\n # # # never enable for an untrusted CDP endpoint.\n # # # Chrome must be version-matched to the bundled\n # # # Playwright; a much newer Chrome can reject the\n # # # CDP handshake. If so, use headless: false to launch\n # # # Playwright's own visible browser instead.\n # # Multi-worker note: browser sessions live in one Gateway worker's memory.\n # # Keep GATEWAY_WORKERS=1 while this tool group is enabled.\n # - name: browser_snapshot\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_snapshot_tool\n # - name: browser_click\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_click_tool\n # - name: browser_type\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_type_tool\n # - name: browser_get_text\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_get_text_tool\n # max_chars: 8000 # Truncation cap for browser_get_text output\n # - name: browser_back\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_back_tool\n # - name: browser_screenshot\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_screenshot_tool\n # - name: browser_close\n # group: browser\n # use: deerflow.community.browser_automation.tools:browser_close_tool\n\n # Web fetch tool (uses Exa)\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry below before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.exa.tools:web_fetch_tool\n # # api_key: $EXA_API_KEY\n\n # Web fetch tool (uses Jina AI reader)\n - name: web_fetch\n group: web\n use: deerflow.community.jina_ai.tools:web_fetch_tool\n timeout: 10\n # Optional proxy for restricted networks / Docker / WSL.\n # Use host.docker.internal instead of 127.0.0.1 when the proxy runs on the host.\n # proxy: $HTTPS_PROXY\n # trust_env: true\n\n # Web fetch tool (uses InfoQuest)\n # - name: web_fetch\n # group: web\n # use: deerflow.community.infoquest.tools:web_fetch_tool\n # # Overall timeout for the entire crawling process (in seconds). Set to positive value to enable, -1 to disable\n # timeout: 10\n # # Waiting time after page loading (in seconds). Set to positive value to enable, -1 to disable\n # fetch_time: 10\n # # Timeout for navigating to the page (in seconds). Set to positive value to enable, -1 to disable\n # navigation_timeout: 30\n\n # Web fetch tool (uses Firecrawl, requires FIRECRAWL_API_KEY)\n # - name: web_fetch\n # group: web\n # use: deerflow.community.firecrawl.tools:web_fetch_tool\n # # api_key: $FIRECRAWL_API_KEY\n\n # Web fetch tool (uses GroundRoute, requires GROUNDROUTE_API_KEY)\n # Fetches a page's extracted text via GroundRoute mode=page.\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry above before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.groundroute.tools:web_fetch_tool\n # # api_key: $GROUNDROUTE_API_KEY\n\n # Web fetch tool (uses fastCRW - Firecrawl-compatible web scraper, single binary,\n # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.)\n # NOTE: Only one web_fetch provider can be active at a time.\n # Comment out the Jina AI web_fetch entry above before enabling this one.\n # - name: web_fetch\n # group: web\n # use: deerflow.community.fastcrw.tools:web_fetch_tool\n # # api_key: $CRW_API_KEY\n # # base_url: https://fastcrw.com/api # default cloud; set to e.g. http://localhost:3000 for self-host\n # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets.\n\n # Image search tool (uses DuckDuckGo)\n # Use this to find reference images before image generation\n - name: image_search\n group: web\n use: deerflow.community.image_search.tools:image_search_tool\n max_results: 5\n\n # Image search tool (uses InfoQuest)\n # - name: image_search\n # group: web\n # use: deerflow.community.infoquest.tools:image_search_tool\n # # Used to limit the scope of image search results, only returns content within the specified time range. Set to -1 to disable time filtering\n # image_search_time_range: 10\n # # Image size filter. Options: \"l\" (large), \"m\" (medium), \"i\" (icon).\n # image_size: \"i\"\n\n # Image search tool (uses Serper - Google Images API, requires SERPER_API_KEY)\n # Serper provides real-time Google Images results. Sign up at https://serper.dev\n # Note: set SERPER_API_KEY in your environment before starting the app.\n # Avoid putting literal API keys in config.yaml; use the $VAR form instead.\n # - name: image_search\n # group: web\n # use: deerflow.community.serper.tools:image_search_tool\n # max_results: 5 # capped at 10 by the Serper provider\n # # api_key: $SERPER_API_KEY # Optional explicit env-var reference\n\n # Image search tool (uses Brave Image Search API, requires BRAVE_SEARCH_API_KEY)\n # Brave provides independent image results and works alongside the Brave web search tool.\n # Note: set BRAVE_SEARCH_API_KEY in your environment before starting the app.\n # Avoid putting literal API keys in config.yaml; use the $VAR form instead.\n # - name: image_search\n # group: web\n # use: deerflow.community.brave.tools:image_search_tool\n # max_results: 5 # capped at 200 by Brave Image Search\n # # country: US\n # # search_lang: en\n # # safesearch: strict\n # # spellcheck: true\n # # api_key: $BRAVE_SEARCH_API_KEY # Optional explicit env-var reference\n\n # File operations tools\n - name: ls\n group: file:read\n use: deerflow.sandbox.tools:ls_tool\n\n - name: read_file\n group: file:read\n use: deerflow.sandbox.tools:read_file_tool\n\n - name: glob\n group: file:read\n use: deerflow.sandbox.tools:glob_tool\n max_results: 200\n\n - name: grep\n group: file:read\n use: deerflow.sandbox.tools:grep_tool\n max_results: 100\n\n - name: write_file\n group: file:write\n use: deerflow.sandbox.tools:write_file_tool\n\n - name: str_replace\n group: file:write\n use: deerflow.sandbox.tools:str_replace_tool\n\n # Bash execution tool\n # Active only when using an isolated shell sandbox or when\n # sandbox.allow_host_bash: true explicitly opts into host bash.\n - name: bash\n group: bash\n use: deerflow.sandbox.tools:bash_tool\n\n# ============================================================================\n# Tool Search Configuration (Deferred Tool Loading)\n# ============================================================================\n# When enabled, MCP tools are not loaded into the agent's context directly.\n# Instead, they are listed by name in the system prompt and discoverable\n# via the tool_search tool at runtime.\n# This reduces context usage and improves tool selection accuracy when\n# multiple MCP servers expose a large number of tools.\n\ntool_search:\n enabled: false\n # When tool_search is enabled, PR1 MCP routing metadata can auto-promote\n # matching deferred MCP tool schemas before a model call. This is the maximum\n # number of matched schemas promoted per model call. Valid range: 1..5.\n auto_promote_top_k: 3\n\n# ============================================================================\n# Tool Output Budget Protection\n# ============================================================================\n# Prevents oversized tool results from blowing the model context window.\n# Outputs exceeding `externalize_min_chars` are persisted to disk and replaced\n# with a compact typed synopsis + file reference. The model can read the full output\n# via read_file. When disk persistence is unavailable, outputs exceeding\n# `fallback_max_chars` are head+tail truncated instead.\n#\n# `exempt_tools` prevents persist→read→persist infinite loops for read tools.\n# `tool_overrides` allows per-tool threshold customization.\n\ntool_output:\n enabled: true\n externalize_min_chars: 12000\n # Sampling budget for the inline raw head/tail sample appended to every\n # typed synopsis; ignored for binary-like output, which carries its own sample.\n preview_head_chars: 2000\n preview_tail_chars: 1000\n fallback_max_chars: 30000\n fallback_head_chars: 8000\n fallback_tail_chars: 3000\n storage_subdir: \".tool-results\"\n exempt_tools:\n - read_file\n - read_file_tool\n # tool_overrides:\n # web_search: 8000\n # bash: 20000\n\n# ============================================================================\n# Suggestions Configuration\n# ============================================================================\n# Configure whether the agent automatically generates follow-up question\n# suggestions at the end of each response.\n\nsuggestions:\n enabled: true\n max_suggestions: 3\n\n\n# ============================================================================\n# Input Polish Configuration\n# ============================================================================\n# Configure whether the composer can rewrite draft input before sending.\n\ninput_polish:\n enabled: true\n # Maximum draft length accepted by /api/input-polish.\n max_chars: 4000\n # Optional fast model for draft polishing. Leave null to use the default chat model.\n # For best UX, set this to your lowest-latency inexpensive model.\n model_name: null\n\n\n# ============================================================================\n# Loop Detection Configuration\n# ============================================================================\n# Detect and interrupt repeated identical tool-call loops.\n# Frequency thresholds are safety limits for repeated use of the same tool type.\n\nloop_detection:\n enabled: true\n warn_threshold: 3\n hard_limit: 5\n window_size: 20\n max_tracked_threads: 100\n tool_freq_warn: 30\n tool_freq_hard_limit: 50\n # Per-tool overrides for tool_freq_warn / tool_freq_hard_limit. Values can be\n # higher or lower than the global defaults. Commonly used to raise thresholds\n # for high-frequency tools like bash in batch workflows (e.g. RNA-seq pipelines)\n # without weakening protection on every other tool.\n # tool_freq_overrides:\n # bash:\n # warn: 150\n # hard_limit: 300\n\n# ============================================================================\n# Tool Progress State Machine Configuration (RFC #3177)\n# ============================================================================\n# Detects tool stagnation and repetition at the (thread, tool) level.\n# Tracks consecutive \"no-new-info\" calls (error, partial_success, near-duplicate success).\n# Three transition paths (determined by deerflow_tool_meta.recoverable_by_model):\n# recoverable=true (no_results, not_found, permission): ACTIVE → WARNED (terminal; hint re-injected each call)\n# recoverable=false (rate_limited, transient): ACTIVE → WARNED → BLOCKED after warn_escalation_count more\n# recoverable=false + action=stop (auth, config): ACTIVE → BLOCKED immediately\n# Requires ToolErrorHandlingMiddleware to be active (always on).\n\n# tool_progress:\n# enabled: false\n# stagnation_threshold: 3 # Consecutive problems before WARNED\n# warn_escalation_count: 2 # More problems after WARNED before BLOCKED\n# inject_assessment: true\n# jaccard_similarity_threshold: 0.8 # Word-set similarity threshold for near-duplicate detection\n# min_word_count_for_similarity: 10 # Min unique words to apply Jaccard check\n# max_tracked_threads: 100\n# exempt_tools:\n# - ask_clarification\n# - write_todos\n# - present_files\n# - task\n\n# ============================================================================\n# Read-Before-Write File Gate (issue #3857)\n# ============================================================================\n# Blocks write_file (append / overwrite of an existing file) and str_replace\n# unless the agent has read the file's current version first; any write\n# invalidates earlier reads, forcing a re-read between consecutive edits.\n# Deterministic guardrail against blind duplicate appends in long tasks.\n\nread_before_write:\n enabled: true\n\n# ============================================================================\n# Provider Safety Termination Configuration\n# ============================================================================\n# Intercept AIMessages where the provider stopped generation for safety reasons\n# (e.g. OpenAI finish_reason='content_filter', Anthropic stop_reason='refusal',\n# Gemini finish_reason='SAFETY') while still returning tool_calls. The\n# tool_calls in such responses are typically truncated/unreliable and must\n# not be executed. See issue #3028 for the full failure mode.\n#\n# Detectors are loaded by class path via reflection (same pattern as\n# guardrails / models / tools). The built-in set covers OpenAI-compatible\n# content_filter, Anthropic refusal, and Gemini SAFETY/BLOCKLIST/\n# PROHIBITED_CONTENT/SPII/RECITATION.\n\nsafety_finish_reason:\n enabled: true\n # Leave `detectors` unset to use the built-in detector set. Set to a\n # non-empty list to fully override (use `enabled: false` to disable instead\n # of providing an empty list).\n #\n # Example — extend the OpenAI-compatible detector for a Chinese provider\n # whose gateway uses a non-standard finish_reason token:\n # detectors:\n # - use: deerflow.agents.middlewares.safety_termination_detectors:OpenAICompatibleContentFilterDetector\n # config:\n # finish_reasons: [\"content_filter\", \"sensitive\", \"risk_control\"]\n # - use: deerflow.agents.middlewares.safety_termination_detectors:AnthropicRefusalDetector\n # - use: deerflow.agents.middlewares.safety_termination_detectors:GeminiSafetyDetector\n #\n # Example — add a custom detector for an in-house provider:\n # detectors:\n # - use: my_company.deerflow_ext:WenxinSafetyDetector\n # config:\n # error_codes: [336003, 17, 18]\n\n# ============================================================================\n# Sandbox Configuration\n# ============================================================================\n# Choose between local sandbox (direct execution) or Docker-based AIO sandbox\n\n# Option 1: Local Sandbox (Default)\n# Executes commands directly on the host machine\nuploads:\n # Application-level upload limits enforced by the gateway and exposed to the\n # frontend before file selection.\n max_files: 10\n max_file_size: 52428800 # 50 MiB\n max_total_size: 104857600 # 100 MiB\n # Automatic Office/PDF conversion runs on the backend host before sandbox\n # isolation applies. Keep this disabled unless uploads come from a fully\n # trusted source and you intentionally accept host-side parser risk.\n auto_convert_documents: false\n # Controls which PDF-to-Markdown converter is used whenever PDF conversion\n # runs. Automatic upload conversion is gated separately by\n # auto_convert_documents.\n # auto — prefer pymupdf4llm when installed; fall back to MarkItDown for\n # image-based or encrypted PDFs (recommended default).\n # pymupdf4llm — always use pymupdf4llm (must be installed: uv add pymupdf4llm).\n # Better heading/table extraction; faster on most files.\n # markitdown — always use MarkItDown (original behaviour, no extra dependency).\n pdf_converter: auto\n\nsandbox:\n use: deerflow.sandbox.local:LocalSandboxProvider\n # Host bash execution is disabled by default because LocalSandboxProvider is\n # not a secure isolation boundary for shell access. Enable only for fully\n # trusted, single-user local workflows.\n allow_host_bash: false\n # Optional: Mount additional host directories into the sandbox.\n # Each mount maps a host path to a virtual container path accessible by the agent.\n # Note: with LocalSandboxProvider under `make up` (docker-compose), host_path is\n # checked from inside the deer-flow-gateway container — you must also bind-mount\n # the same directory into services.gateway.volumes in docker/docker-compose.yaml\n # for this mount to take effect (see issue #3244).\n # mounts:\n # - host_path: /home/user/my-project # Absolute path; see note above for Docker mode\n # container_path: /mnt/my-project # Virtual path inside the sandbox\n # read_only: true # Whether the mount is read-only (default: false)\n\n # Tool output truncation limits (characters).\n # bash uses middle-truncation (head + tail) since errors can appear anywhere in the output.\n # read_file and ls use head-truncation since their content is front-loaded.\n # Set to 0 to disable truncation.\n bash_output_max_chars: 20000\n read_file_output_max_chars: 50000\n ls_output_max_chars: 20000\n\n # Maximum wall-clock seconds a single host bash command may run before it is\n # terminated (process group and all). A blocking foreground command — e.g. a\n # server started without backgrounding — is killed after this long so the\n # agent's turn cannot hang. Start long-lived processes in the background with\n # output redirected (e.g. `your-command > /tmp/server.log 2>&1 &`) when you\n # need logs; unredirected background output is drained with bounded capture\n # and excess output is discarded.\n bash_command_timeout: 600\n\n# Option 2: Container-based AIO Sandbox\n# Executes commands in isolated containers (Docker or Apple Container)\n# On macOS: Automatically prefers Apple Container if available, falls back to Docker\n# On other platforms: Uses Docker\n# Uncomment to use:\n# sandbox:\n# use: deerflow.community.aio_sandbox:AioSandboxProvider\n#\n# # Optional: Container image to use (works with both Docker and Apple Container)\n# # Default: enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest\n# # The mirror's `:latest` tag is frozen on an old pre-1.9.3 digest that lacks\n# # the /v1/bash/* routes required-secrets skills need (see #3921/#3922), so\n# # pin an explicit version instead — recommended: 1.11.0 (multi-arch, works\n# # on both x86_64 and arm64). Custom images should extend the default image\n# # or implement the same AIO sandbox HTTP API used by agent-sandbox. See\n# # backend/docs/CONFIGURATION.md.\n# # image: enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0\n#\n# # Optional: Base port for sandbox containers (default: 8080)\n# # port: 8080\n\n# # Optional: Maximum number of concurrent sandbox containers (default: 3)\n# # When the limit is reached the least-recently-used sandbox is evicted to\n# # make room for new ones. Use a positive integer here; omit this field to use the default.\n# # replicas: 3\n#\n# # Optional: Prefix for container names (default: deer-flow-sandbox)\n# # container_prefix: deer-flow-sandbox\n#\n# # Optional: Override whether the sandbox already sees the gateway's\n# # thread workspace/uploads/outputs through shared mounts.\n# # Omit this field to auto-detect from the backend (local containers: true;\n# # remote/provisioner backends: false). Set true only when the deployment\n# # guarantees both sides use the same thread user-data directories, such as\n# # a correctly aligned shared PVC, NFS volume, hostPath, or bind mount.\n# # true skips per-upload sandbox acquire/sync; false forces explicit sync.\n# # thread_data_mounts: true\n#\n# # Optional: Additional mount directories from host to container\n# # NOTE: Skills directory is automatically mounted from skills.path to skills.container_path\n# # mounts:\n# # # Other custom mounts\n# # - host_path: /path/on/host\n# # container_path: /home/user/shared\n# # read_only: false\n# #\n# # # DeerFlow will surface configured container_path values to the agent,\n# # # so it can directly read/write mounted directories such as /home/user/shared\n#\n# # Optional: Environment variables to inject into the sandbox container\n# # Values starting with $ will be resolved from host environment variables\n# # environment:\n# # NODE_ENV: production\n# # DEBUG: \"false\"\n# # API_KEY: $MY_API_KEY # Reads from host's MY_API_KEY env var\n# # DATABASE_URL: $DATABASE_URL # Reads from host's DATABASE_URL env var\n#\n# # Optional: Cross-instance container ownership (issue #4206).\n# #\n# # Gateway instances share sandbox containers but each keeps its own in-memory\n# # warm pool. Without shared ownership state, one instance's startup\n# # reconciliation adopts a container another instance is actively using and\n# # later idle-destroys it — tool calls then fail with 502 / connection refused.\n# #\n# # Single gateway instance? Leave this out; `memory` is the default and the\n# # cross-instance kill cannot happen.\n# #\n# # MULTIPLE gateway instances / workers sharing one container backend\n# # (load-balanced deployments, Docker Compose) MUST set type: redis. Docker\n# # Compose already sets DEER_FLOW_STREAM_BRIDGE_REDIS_URL, which is taken as\n# # proof the deployment is multi-instance, so redis ownership is inferred even\n# # if this section is omitted.\n# #\n# # The redis ownership store requires the optional `redis` extra. It is\n# # auto-detected from this section on `make dev` and always installed in the\n# # Docker image. To install it manually:\n# # cd backend && uv sync --all-packages --extra redis\n# #\n# # ownership:\n# # type: memory # single gateway instance only\n# #\n# # ownership:\n# # type: redis # required for multi-instance / load-balanced\n# # redis_url: redis://redis:6379/0\n# # renewal_interval_seconds: 30 # how often an owner refreshes its leases\n# # ttl_multiplier: 4 # lease TTL = interval x this (min 2, so a\n# # # single missed renewal cannot expire a live\n# # # owner). Liveness is deliberately independent\n# # # of idle_timeout: renewal keeps running even\n# # # at idle_timeout: 0.\n# # key_prefix: deerflow:sandbox:owner\n# #\n# # NOTE: the redis ownership store is fail-closed, matching the stream bridge's\n# # fail-hard policy. Redis.from_url is lazy so a down Redis does not block\n# # startup, but a sandbox whose ownership cannot be published is not handed out\n# # — acquiring raises instead. The alternative (proceed unowned) is exactly the\n# # #4206 cross-instance kill. Run Redis with HA / a restart policy.\n# #\n# # NOTE: the other boundary is the lease TTL (renewal_interval_seconds x\n# # ttl_multiplier). A Redis outage longer than the TTL can let a reconciling\n# # instance adopt a live owner's container: a lapsed lease is indistinguishable\n# # from a dead owner, so one TTL of adoption grace is the whole safety margin.\n# # Size the TTL against your Redis availability target (HA Redis keeps this\n# # window out of reach).\n\n# Option 3: BoxLite micro-VM Sandbox\n# Runs each sandbox as a BoxLite micro-VM. Released boxes stay in an in-process\n# warm pool and can be reclaimed by the same user/thread without a cold start.\n# Requires the boxlite runtime and host virtualization support (KVM on Linux,\n# Hypervisor.framework on macOS).\n# sandbox:\n# use: deerflow.community.boxlite:BoxliteProvider\n# image: python:3.12-slim\n#\n# # Optional: Per-box memory and CPU limits.\n# # memory_mib: 1024\n# # cpus: 2\n#\n# # Optional: Maximum active + warm BoxLite VMs per gateway process (default: 3).\n# # Active boxes are never evicted; only warm-pool boxes are stopped to make room.\n# # replicas: 3\n#\n# # Optional: Seconds before an idle warm-pool VM is stopped (default: 600).\n# # Set to 0 to keep warm VMs until shutdown or replica eviction.\n# # idle_timeout: 600\n#\n# # Optional: Skip the reclaim health check for very recently released VMs.\n# # Default 0.0 keeps reliability-first validation before warm reuse.\n# # health_check_skip_seconds: 0.0\n#\n# # Optional: Environment variables to inject into every command.\n# # environment:\n# # PYTHONUNBUFFERED: \"1\"\n#\n# Option 4: Provisioner-managed AIO Sandbox (docker-compose-dev)\n# Each sandbox_id gets a dedicated Pod in k3s, managed by the provisioner.\n# Recommended for production or advanced users who want better isolation and scalability.:\n# sandbox:\n# use: deerflow.community.aio_sandbox:AioSandboxProvider\n# provisioner_url: http://provisioner:8002\n# # API key for provisioner authentication. Must match PROVISIONER_API_KEY\n# # set on the provisioner container. Both sides must have the same value set;\n# # the provisioner rejects all /api/* requests when PROVISIONER_API_KEY is unset.\n# # Generate a strong key: openssl rand -hex 32\n# # provisioner_api_key: $PROVISIONER_API_KEY\n# # Note: provisioner-created Pods use the provisioner's SANDBOX_IMAGE\n# # environment variable, not sandbox.image from this config file.\n\n# Option 5: Tenki cloud microVM Sandbox\n# Runs each sandbox as an isolated Tenki cloud microVM. Released sandboxes stay\n# in an in-process warm pool and are reclaimed by the same user/thread without a\n# cold start. Requires the optional SDK: pip install \"deerflow-harness[tenki]\".\n# sandbox:\n# use: deerflow.community.tenki:TenkiSandboxProvider\n# # api_key: $TENKI_API_KEY # falls back to TENKI_API_KEY / TENKI_AUTH_TOKEN env var\n# # base_url: https://tenki.cloud # optional; SDK default when omitted\n# # image: my-base-image # optional; Tenki account default base image when omitted\n# # project_id: proj_... # optional; auto-selected if the account has exactly one\n# # workspace_id: ws_... # optional; auto-selected if the account has exactly one\n# # cpu_cores: 2 # optional per-sandbox vCPUs\n# # memory_mb: 2048 # optional per-sandbox memory\n# # replicas: 3 # active + warm microVM cap per gateway process\n# # idle_timeout: 600 # warm microVM idle seconds before terminate; 0 disables\n# # max_duration: 14400 # Tenki sandbox lifetime in seconds; 0 uses the account default\n# # sticky: false # pin the microVM to its host (only matters with pause/resume)\n# # home_dir: /home/tenki # writable dir backing /mnt/user-data\n# # environment: # injected into every command (and as create-time env)\n# # PYTHONUNBUFFERED: \"1\"\n\n# ============================================================================\n# Subagents Configuration\n# ============================================================================\n# Configure timeouts for subagent execution\n# Subagents are background workers delegated tasks by the lead agent\n\n# subagents:\n# # Default timeout (seconds) for built-in subagents (default: 1800 = 30 min).\n# # Custom agents use their own timeout_seconds (default 900) unless overridden.\n# timeout_seconds: 1800\n# # Optional global max-turn override for all subagents.\n# # Built-in defaults: general-purpose=150, bash=60. Leave unset to keep them.\n# # max_turns: 120\n#\n# # Total number of subagent delegations allowed in one lead-agent run.\n# # This is a deterministic backstop against repeated planning checkpoints\n# # launching legal-sized batches forever. The default 6 allows two full\n# # batches at the default concurrency of 3. Valid config range: 1-50.\n# # Per-request runtime context can temporarily override this with\n# # `max_total_subagents`, clamped to the same 1-50 range.\n# max_total_per_run: 6\n#\n# # Per-run token ceiling for subagents (#3875 Phase 2). A backstop against a\n# # subagent that burns tokens on trivial work. At the hard-stop threshold the\n# # in-flight turn is capped (tool calls stripped, finish_reason forced to\n# # \"stop\") so the run completes naturally with a final answer; the result is\n# # stamped `completed` + `subagent_stop_reason=token_capped` so the lead and\n# # UI can tell a budget-capped completion from a clean one. The 2,000,000\n# # default is a generous ceiling — lower it to tighten cost controls. A\n# # per-agent `token_budget` override (see `agents:` below) wins over this.\n# # token_budget:\n# # enabled: true\n# # max_tokens: 2000000\n# # warn_threshold: 0.7 # log a warning once this fraction of the budget is spent\n#\n# # Optional per-agent overrides (applies to both built-in and custom agents)\n# agents:\n# general-purpose:\n# timeout_seconds: 2700 # 45 minutes for very long deep-research tasks\n# max_turns: 250 # raise above the 150 default for very deep tasks\n# # token_budget: # per-agent override of the global token_budget above\n# # max_tokens: 3000000 # raise the ceiling for deep-research tasks\n# # model: qwen3:32b # Use a specific model (default: inherit from lead agent)\n# # skills: # Skill discovery/activation allowlist (default: all enabled)\n# # - web-search\n# # - data-analysis\n# bash:\n# timeout_seconds: 300 # 5 minutes for quick command execution\n# max_turns: 80\n# # skills: [] # No discoverable/activatable skills for bash agent\n#\n# # Custom subagent types: define specialized agents with their own prompts,\n# # tools, skills, and model configuration. Custom agents are available via\n# # the `task` tool alongside built-in types (general-purpose, bash).\n# # custom_agents:\n# # analysis:\n# # description: \"Data analysis specialist for processing datasets and generating insights\"\n# # system_prompt: |\n# # You are a data analysis subagent. Focus on:\n# # - Processing and analyzing datasets\n# # - Generating visualizations\n# # - Providing statistical insights\n# # tools: # Tool whitelist (null = inherit all)\n# # - bash\n# # - read_file\n# # - write_file\n# # skills: # Skill discovery/activation allowlist (null = all, [] = none)\n# # - data-analysis\n# # - visualization\n# # model: inherit # 'inherit' uses parent's model\n# # max_turns: 80\n# # timeout_seconds: 600\n#\n# # Model override: by default, subagents inherit the lead agent's model.\n# # Set `model` to use a different model (e.g., a local Ollama model for cost savings).\n# # The model name must match a name defined in the `models:` section above.\n\n# ============================================================================\n# ACP Agents Configuration\n# ============================================================================\n# Configure external ACP-compatible agents for the built-in `invoke_acp_agent` tool.\n\n# acp_agents:\n# claude_code:\n# # DeerFlow expects an ACP adapter here. The standard `claude` CLI does not\n# # speak ACP directly. Install `claude-agent-acp` separately or use:\n# command: npx\n# args: [\"-y\", \"@zed-industries/claude-agent-acp\"]\n# description: Claude Code for implementation, refactoring, and debugging\n# model: null\n# # auto_approve_permissions: false # Set to true to auto-approve ACP permission requests\n# # timeout_seconds: 1800 # Abort + kill the subprocess if it doesn't respond in time (default: 1800 = 30 min)\n# # env: # Optional: inject environment variables into the agent subprocess\n# # ANTHROPIC_API_KEY: $ANTHROPIC_API_KEY # $VAR resolves from host environment\n#\n# codex:\n# # DeerFlow expects an ACP adapter here. The standard `codex` CLI does not\n# # speak ACP directly. Install `codex-acp` separately or use:\n# command: npx\n# args: [\"-y\", \"@zed-industries/codex-acp\"]\n# description: Codex CLI for repository tasks and code generation\n# model: null\n# # auto_approve_permissions: false # Set to true to auto-approve ACP permission requests\n# # env: # Optional: inject environment variables into the agent subprocess\n# # OPENAI_API_KEY: $OPENAI_API_KEY # $VAR resolves from host environment\n\n# ============================================================================\n# Skills Configuration\n# ============================================================================\n# Configure skills directory for specialized agent workflows\n\nskills:\n # Path to skills directory on the host (relative to project root or absolute)\n # Default: skills under the project root\n # Override with DEER_FLOW_SKILLS_PATH when this field is omitted.\n # Uncomment to customize:\n # path: /absolute/path/to/custom/skills\n\n # Path where skills are mounted in the sandbox container\n # This is used by the agent to access skills in both local and Docker sandbox\n # Default: /mnt/skills\n container_path: /mnt/skills\n\n # Deferred skill discovery (default: false)\n # When enabled, only skill names appear in the system prompt ().\n # The LLM discovers skill details on demand via the describe_skill tool.\n # This keeps the system prompt compact and prefix-cache friendly when many\n # skills are installed.\n # deferred_discovery: true\n\n# ============================================================================\n# SkillScan Configuration\n# ============================================================================\n# Native deterministic skill safety scanning. This runs before the LLM skill\n# scanner on skill install/update and agent-managed skill writes.\nskill_scan:\n # Set false to disable the new deterministic analyzers (nested-archive,\n # secret-pattern, and other content-level checks). Safe archive extraction\n # (path traversal, symlinks, executable-binary, total-size, and entry-count\n # limits) and the LLM skill scanner still run unconditionally.\n enabled: true\n\n# Note: To restrict which skills are loaded for a specific custom agent,\n# define a `skills` list in that agent's `config.yaml` (e.g. `agents/my-agent/config.yaml`):\n# - Omitted or null: load all globally enabled skills (default)\n# - []: disable all skills for this agent\n# - [\"skill-name\"]: load only specific skills\n\n# ============================================================================\n# Title Generation Configuration\n# ============================================================================\n# Automatic conversation title generation settings\n\ntitle:\n enabled: true\n max_words: 6\n max_chars: 60\n model_name: null # null = fast local fallback; set a model name to use LLM title generation\n\n# ============================================================================\n# Summarization Configuration\n# ============================================================================\n# Automatically summarize conversation history when token limits are approached\n# This helps maintain context in long conversations without exceeding model limits\n\nsummarization:\n enabled: true\n\n # Model to use for summarization.\n # null = summarize with the model the run actually uses (the lead run's model, a\n # subagent's own model, or a thread's custom-agent model), NOT models[0].\n # set = that model generates; if its provider fails, compaction falls back to the\n # run's own model so a broken summary provider cannot disable compaction.\n # Recommended: Use a lightweight, cost-effective model like \"gpt-4o-mini\" or similar\n model_name: null\n\n # Trigger conditions - at least one required\n # Summarization runs when ANY threshold is met (OR logic)\n # You can specify a single trigger or a list of triggers\n trigger:\n # Trigger when token count reaches 32000\n - type: tokens\n value: 32000\n # Uncomment to also trigger when message count reaches 50\n # - type: messages\n # value: 50\n # Uncomment to trigger when 80% of model's max input tokens is reached\n # - type: fraction\n # value: 0.8\n\n # Context retention policy after summarization\n # Specifies how much recent history to preserve\n keep:\n # Keep the most recent 10 messages (recommended)\n type: messages\n value: 10\n # Alternative: Keep specific token count\n # type: tokens\n # value: 3000\n # Alternative: Keep percentage of model's max input tokens\n # type: fraction\n # value: 0.3\n\n # Maximum tokens to keep when preparing messages for summarization\n # Set to null to skip trimming (not recommended for very long conversations)\n trim_tokens_to_summarize: 15564\n\n # Custom summary prompt template (null = use default LangChain prompt)\n # The prompt should guide the model to extract important context\n summary_prompt: null\n\n # Loaded SKILL.md references (read_file calls under skills.container_path) are\n # captured into the durable skill_context channel and re-injected after\n # compaction as name/path/description reminders. The full skill body is not\n # persisted; the agent should re-read the file before applying instructions.\n # Tool names counted as skill reads:\n # Legacy preserve_recent_skill_* summarization settings are no longer used;\n # skill retention is handled by this durable reference channel instead. Set\n # this list to [] to disable durable skill-reference capture.\n skill_file_read_tool_names:\n - read_file\n - read\n - view\n - cat\n\n# ============================================================================\n# Memory Configuration\n# ============================================================================\n# Global memory mechanism (pluggable + self-contained).\n#\n# Shared fields (host level, backend-agnostic):\n# enabled - Master switch for the memory mechanism (call-site gate)\n# injection_enabled - Whether to inject memory into the system prompt (call-site gate)\n# shutdown_flush_timeout_seconds - Hard budget (s) to drain pending updates on Gateway graceful shutdown (default: 30)\n# manager_class - Backend selector: registered name (deermem/mem0/noop/openviking) or dotted path\n# backend_config - Backend-private config dict (passthrough; each backend self-interprets)\n#\n# DeerMem-private fields live under ``backend_config`` (NOT at the memory: top level):\n# storage_path - Data root. Empty = deer-flow base_dir (factory injects absolute\n# runtime_home). Absolute path = that root. Relative = CWD-relative.\n# model - LLM config for memory extraction: {provider, model, api_key, base_url,\n# temperature}. Omit all fields = no extraction (non-LLM ops still work).\n# debounce_seconds - Debounce wait before processing queued updates (default: 30)\n# max_facts - Maximum facts to store (default: 100)\n# fact_confidence_threshold - Minimum confidence for storing facts (default: 0.7)\n# max_injection_tokens - Token budget for memory injection (default: 2000)\n# token_counting - tiktoken (accurate, network on first use) or char (network-free)\n# guaranteed_categories - Fact categories always injected (default: [\"correction\"])\n# guaranteed_token_budget - Token ceiling for guaranteed categories (default: 500)\n# staleness_review_enabled - Enable staleness pruning (default: true)\n# staleness_age_days - Age threshold for staleness candidates (default: 90)\n# staleness_min_candidates - Minimum stale facts to trigger review (default: 3)\n# staleness_max_removals_per_cycle - Safety cap on removals per cycle (default: 10)\n# staleness_protected_categories - Categories exempt from staleness review (default: [\"correction\"])\n# staleness_max_lifetime_multiplier - Creation-time cap multiplier for a fact LLM-assigned\n# expected_valid_days; new facts clamped to\n# staleness_age_days x multiplier. Default 20.0\n# (90 x 20 = 1800d ~ 5 years) supports the very-stable\n# prompt tier. (default: 20.0, range: 1.0-100.0)\n# staleness_max_extension_days - Absolute ceiling (in days) on expected_valid_days after a\n# lifetime extension (staleFactsToExtend). Prevents timedelta\n# overflow and LLM misfire from permanently deferring a fact.\n# (default: 3650, range: 90-36500)\nmemory:\n enabled: true\n injection_enabled: true # Whether to inject memory into system prompt\n # Hard budget (seconds) to drain pending memory updates on Gateway graceful\n # shutdown. Each pending item does one LLM call, so large IM batches may need\n # more. Must fit inside the pod's K8s terminationGracePeriodSeconds (channel\n # stop + this drain + buffer) or K8s SIGKILLs the drain -- set that on the\n # gateway Helm deployment (see deploy/helm/deer-flow). Default 30s.\n shutdown_flush_timeout_seconds: 30.0\n # Memory backend selector. Either a registered backend name (matching a\n # backends// folder that exposes MANAGER_CLASS, e.g. deermem / mem0 / noop)\n # or a dotted import path to a MemoryManager subclass.\n manager_class: deermem\n # Memory operation mode:\n # middleware (default) - passive background extraction after each turn.\n # tool - experimental opt-in; the model calls memory_search/memory_add/\n # memory_update/memory_delete directly. This gives the model agency over\n # memory writes, but effectiveness depends on model tool-use behavior.\n # Normally only one mode runs at a time. A backend that needs conversation-\n # level extraction may retain passive writes in tool mode while still\n # exposing query-aware search (mem0 does this). Tool calls go through the\n # active MemoryManager; unsupported fact CRUD returns a JSON error.\n mode: middleware\n # Backend-private config (a dict), passed verbatim to the backend __init__.\n # Each backend self-interprets it (DeerMem parses it into DeerMemConfig).\n #\n # OpenViking HTTP example (replace this DeerMem backend_config block when\n # manager_class is openviking; OpenViking currently supports middleware mode):\n #\n # backend_config:\n # base_url: http://openviking:1933\n # auth_mode: trusted\n # account: deerflow\n # api_key_env: OPENVIKING_API_KEY\n # max_connections: 100\n # max_keepalive_connections: 20\n # max_seen_message_ids: 512 # recent-ID fallback for compacted histories\n # startup_policy: fail_fast\n # failure_policy:\n # read: fail_open\n # write: log_and_drop\n # retrieval:\n # top_k: 8\n # score_threshold: 0.25\n # max_injection_chars: 12000\n #\n # For a host-installed OpenViking used by Docker DeerFlow, set base_url to\n # http://host.docker.internal:1933 and allow_insecure_http: true. The bundled\n # optional Compose overlay uses the internal http://openviking:1933 address.\n backend_config:\n storage_path: \"\" # empty = deer-flow base_dir (factory injects absolute runtime_home); a non-empty path is the root DIRECTORY (per-user memory under {storage_path}/users/{uid}/memory.json)\n storage_class: file # file (default) or a dotted MemoryStorage class path; invalid persistent backends fail fast\n strict_user_scope: false # set true in authenticated deployments after all callers propagate user_id\n manifest_filename: memory.json # user-global JSON: version/revision/time + user/history only; no facts or fact index\n file_lock_timeout_seconds: 10 # per-scope cross-process advisory lock timeout (single-machine local filesystem)\n retrieval_adapter: fts5 # fts5 (default), empty to disable, or a dotted RetrievalPort factory(config)\n debounce_seconds: 30 # Wait time before processing queued updates\n # Backpressure cap on pending items. 0 = unlimited. At the cap, new\n # non-signal updates are rejected (QueueFull); signal updates are always\n # admitted so important memories are never shed.\n queue_max_depth: 1000\n model: # LLM for memory extraction; omit all fields = no extraction (non-LLM ops still work; an update raises)\n # provider: openai\n # model: gpt-4o-mini\n # api_key: $OPENAI_API_KEY\n # base_url: # optional, for OpenAI-compatible gateways (e.g. DeepSeek)\n # temperature: # optional\n max_facts: 100 # Maximum number of facts to store\n fact_confidence_threshold: 0.7 # Minimum confidence for storing facts\n max_injection_tokens: 2000 # Maximum tokens for memory injection\n # Token counting strategy for memory-injection budgeting:\n # tiktoken (default) - accurate, but the encoding BPE data may be\n # downloaded from a public network endpoint on first use. In\n # network-restricted environments this download can block for a long\n # time (see issues #3402 / #3429). Pre-cache the encoding or set this\n # to \"char\" to avoid it.\n # char - network-free CJK-aware character-based estimate; never touches\n # tiktoken. Slightly less precise budgeting, zero network I/O.\n token_counting: tiktoken\n # Guaranteed injection: fact categories that bypass the regular token budget\n # and draw from a reserved allowance, so high-signal corrections (e.g.\n # \"don use pip, use uv\") survive even when the budget is tight.\n guaranteed_categories:\n - correction\n guaranteed_token_budget: 500\n # Staleness review: periodically prune aged facts that may no longer reflect\n # the user current situation. When triggered, the LLM reviews facts older\n # than their individual review window (expected_valid_days, or\n # staleness_age_days as fallback) during the normal memory-update call (same\n # LLM invocation - no extra API call) and decides KEEP, REMOVE, or EXTEND\n # for each. The LLM assigns expected_valid_days when creating a fact; EXTEND\n # (staleFactsToExtend) recalibrates that window at review time.\n staleness_review_enabled: true\n staleness_age_days: 90\n staleness_min_candidates: 3\n staleness_max_removals_per_cycle: 10\n staleness_protected_categories:\n - correction\n staleness_max_lifetime_multiplier: 20.0 # creation cap = staleness_age_days x multiplier (90 x 20 = 1800d)\n staleness_max_extension_days: 3650 # absolute ceiling on extended expected_valid_days (~10 years)\n # Memory consolidation (opt-in, lossy: source facts are replaced by a\n # synthesized one; only consolidatedFrom IDs are kept). Runs in the same\n # memory-update LLM call as extraction/staleness - no extra API cost.\n # consolidation_enabled defaults to false because consolidation is lossy.\n consolidation_enabled: false # set true to opt into fact consolidation\n consolidation_min_facts: 8 # min facts in one category to trigger review (3-30)\n consolidation_max_groups_per_cycle: 3 # max groups merged per update cycle (1-10)\n consolidation_max_sources: 8 # max source facts per consolidation group (2-20)\n # extraction_callback is a host-injected post-extraction observability hook\n # (token usage, facts accepted/rejected, rejection rate). The factory\n # injects a logging default; set programmatically to emit a Langfuse span.\n # Message processing (externalized patterns / prompt templates):\n # patterns_dir - dir with correction.yaml / reinforcement.yaml overriding\n # the bundled signal-detection patterns. None (default) =\n # bundled core/message_patterns/. When explicitly set, both\n # files must exist (typos or missing mounts raise an error).\n # prompts_dir - dir with custom memory-extraction prompt templates\n # (memory_update.chat.yaml, staleness_review.yaml,\n # consolidation.yaml, fact_extraction.yaml). None (default)\n # = bundled core/prompts/. Supports per-agent subdirectories.\n # patterns_dir: \"\" # empty = bundled defaults\n # prompts_dir: \"\" # empty = bundled defaults\n\n# ============================================================================\n# Custom Agent Management API\n# ============================================================================\n# Controls whether the HTTP gateway exposes custom-agent SOUL/USER.md management.\n# Keep this disabled unless the gateway is behind a trusted authenticated admin boundary.\nagents_api:\n enabled: false\n\n# ============================================================================\n# Skill Self-Evolution Configuration\n# ============================================================================\n# Allow the agent to autonomously create and improve skills in skills/custom/.\nskill_evolution:\n enabled: false # Set to true to allow agent-managed writes under skills/custom\n moderation_model_name: null # Model for LLM-based security scanning (null = use default model)\n security_fail_closed: true # Moderation model unavailable: true blocks all writes; false allows non-executable content with a warning (executable is always blocked)\n\n# ============================================================================\n# Checkpointer Configuration (DEPRECATED — use `database` instead)\n# ============================================================================\n# Legacy standalone checkpointer config. Kept for backward compatibility.\n# Prefer the unified `database` section below, which drives the LangGraph\n# checkpointer, LangGraph Store, and DeerFlow application data (runs,\n# feedback, events) from a single backend setting.\n#\n# If both `checkpointer` and `database` are present, `checkpointer`\n# takes precedence for the LangGraph checkpointer and Store only.\n#\n# checkpointer:\n# type: sqlite\n# connection_string: checkpoints.db\n#\n# checkpointer:\n# type: postgres\n# connection_string: postgresql://user:password@localhost:5432/deerflow\n# # Optional: place LangGraph checkpointer/store tables in this schema.\n# # Leave empty to use the server default search_path, usually public.\n# postgres_schema: deerflow\n\n# ============================================================================\n# Database\n# ============================================================================\n# Unified storage backend for the LangGraph checkpointer, LangGraph Store,\n# and DeerFlow application data (runs, threads metadata, feedback, etc.).\n#\n# backend: memory -- No persistence, data lost on restart\n# backend: sqlite -- Single-node deployment, files in sqlite_dir\n# backend: postgres -- Production multi-node deployment\n#\n# If this section is omitted or empty in config.yaml, DeerFlow uses:\n# backend: sqlite\n# sqlite_dir: .deer-flow/data\n#\n# SQLite mode uses a single deerflow.db file with WAL journal mode\n# for the checkpointer, Store, and application data.\n#\n# Postgres mode: put your connection URL in .env as DATABASE_URL,\n# then reference it here with $DATABASE_URL.\n#\n# Install the driver — Issue #2754 fix lands `UV_EXTRAS` in every code path:\n# Local `make dev` auto-detects from `database.backend: postgres` below\n# and passes `--extra postgres` to `uv sync` on every restart, so\n# the extra is no longer wiped. To opt in explicitly (or layer\n# extras like `postgres,ollama`), set in project-root .env:\n# UV_EXTRAS=postgres\n# Docker dev `make docker-start` reads `UV_EXTRAS` from project-root .env via\n# `env_file`. Set:\n# UV_EXTRAS=postgres\n# Multiple extras (`postgres,ollama`) supported here too — see\n# docker/dev-entrypoint.sh.\n# Docker img build-arg `UV_EXTRAS=postgres,discord docker compose build`\n# supports comma- or whitespace-separated extras at build time\n# (backend/Dockerfile expands them to repeated `--extra` flags).\n#\n# First-time bootstrap (before `make dev`):\n# cd backend && uv sync --all-packages --extra postgres\n# (--all-packages propagates the extra into workspace members — see PR #2584)\n#\n# NOTE: When both `checkpointer` and `database` are configured,\n# `checkpointer` takes precedence for the LangGraph checkpointer and Store;\n# `database` still controls DeerFlow application data.\n# If you use `database`, you can remove the `checkpointer` section.\n# database:\n# backend: sqlite\n# sqlite_dir: .deer-flow/data\n#\n# database:\n# backend: postgres\n# postgres_url: $DATABASE_URL\n# # Optional: place DeerFlow ORM tables and LangGraph checkpointer/store\n# # tables in this schema. Leave empty to use the server default\n# # search_path, usually public.\n# postgres_schema: deerflow\n#\n# postgres_schema only takes effect for new tables. On startup DeerFlow runs\n# CREATE SCHEMA IF NOT EXISTS and pins the connection search_path; it never\n# moves existing tables, so third-party tables in `public` are left untouched.\n#\n# Migrating an EXISTING deployment from `public` to a dedicated schema:\n# 1. Stop the DeerFlow services.\n# 2. Move EVERY DeerFlow-owned table AND the Alembic state into the new schema\n# (run as the DB owner). Missing any table strands its rows: after restart\n# DeerFlow recreates an empty counterpart in the target schema and the old\n# rows stay invisible in `public`. Moving the ORM tables but leaving\n# `alembic_version` behind is especially dangerous -- bootstrap then treats\n# the target schema as unversioned, re-baselines it, and replays migrations.\n# CREATE SCHEMA IF NOT EXISTS deerflow;\n# -- Application ORM tables:\n# ALTER TABLE public.runs SET SCHEMA deerflow;\n# ALTER TABLE public.run_events SET SCHEMA deerflow;\n# ALTER TABLE public.threads_meta SET SCHEMA deerflow;\n# ALTER TABLE public.feedback SET SCHEMA deerflow;\n# ALTER TABLE public.users SET SCHEMA deerflow;\n# ALTER TABLE public.agents SET SCHEMA deerflow;\n# -- IM channel tables:\n# ALTER TABLE public.channel_connections SET SCHEMA deerflow;\n# ALTER TABLE public.channel_credentials SET SCHEMA deerflow;\n# ALTER TABLE public.channel_oauth_states SET SCHEMA deerflow;\n# ALTER TABLE public.channel_conversations SET SCHEMA deerflow;\n# -- Scheduled-task tables:\n# ALTER TABLE public.scheduled_tasks SET SCHEMA deerflow;\n# ALTER TABLE public.scheduled_task_runs SET SCHEMA deerflow;\n# -- Alembic migration state (REQUIRED -- see note above):\n# ALTER TABLE public.alembic_version SET SCHEMA deerflow;\n# The DeerFlow-owned set grows over time; discover any tables the list\n# above misses (as the DeerFlow DB owner/role) with:\n# SELECT table_name FROM information_schema.tables WHERE table_schema='public';\n# LangGraph checkpointer/store table names also vary by version -- list them\n# the same way, then ALTER ... SET SCHEMA deerflow for each (e.g. checkpoints,\n# checkpoint_blobs, checkpoint_writes, checkpoint_migrations, store,\n# store_migrations).\n# 3. Set postgres_schema: deerflow here and restart.\n# 4. Verify with `SHOW search_path;` and a smoke test (create a run, read\n# its history). The `public` schema should gain no new DeerFlow tables.\ndatabase:\n backend: sqlite\n sqlite_dir: .deer-flow/data\n\n# ============================================================================\n# Inbound webhook dedupe storage (cross-pod redelivery dedup — issue #4120)\n# ============================================================================\n# Where the ChannelManager records inbound webhook dedupe state.\n#\n# backend: auto -- (default) Postgres whenever database.backend=postgres,\n# including multi-replica K8s where each pod runs a single\n# worker but shares one DB. Otherwise an in-process memory\n# store (single-DB, single-pod).\n# NOTE: multi-replica deployments must use a Postgres DB;\n# if database.backend is sqlite/memory, 'auto' falls back to\n# the per-pod memory store and cross-pod dedupe is disabled.\n# backend: memory -- In-process store only. Per-pod: a webhook redelivered to\n# a DIFFERENT replica is NOT deduped. Not recommended for\n# multi-worker deployments (logs a startup WARNING).\n# backend: postgres -- Share dedupe state across pods via the application DB.\n# REQUIRED for any multi-replica deployment. If\n# database.backend is not 'postgres', a WARNING is logged and\n# the store falls back to the in-process memory store.\n#\n# dedupe_storage:\n# backend: auto\n\n # Recycle app ORM PostgreSQL connections before the environment's idle cutoff.\n pool_recycle: 300\n # App ORM PostgreSQL command timeout. Set to null to disable it or raise it\n # for intentionally long commands.\n command_timeout: 30\n # Restart required. Use one value across every process sharing this database.\n # full: current full-message checkpoints; delta: DeltaChannel for messages.\n checkpoint_channel_mode: full\n # Delta-mode tuning (only applies when checkpoint_channel_mode is delta).\n # Restart required, like the mode itself: the cadence is compiled into each\n # graph's channel table. All processes sharing this database must agree.\n # snapshot_frequency: full messages snapshot every N per-step writes\n # (higher = smaller checkpoints, slower materialization).\n checkpoint_delta:\n snapshot_frequency: 10\n # Size caps for the compiled checkpoint graph caches (per process).\n # Hot-reloadable, no restart needed: the value is re-read on each eviction\n # check and only changes when a cache evicts, never graph semantics.\n checkpoint_graph_cache:\n # Gateway thread-state accessor graphs (per assistant/mode/cadence).\n accessor_graph_max: 64\n # Delta-mode checkpoint history cache (only used when checkpoint_channel_mode: delta).\n # Pure performance policy: safe to differ across workers, never frozen.\n # checkpoint_cache:\n # type: memory # memory | redis (redis is Gateway/async only)\n # max_entries: 128 # 0 disables the cache\n # redis_url: null # or DEER_FLOW_CHECKPOINT_CACHE_REDIS_URL / REDIS_URL\n # ttl_seconds: 86400 # redis leak safety net; bounds residual copies if a\n # # thread-delete purge fails (purge itself is immediate).\n # # 0 explicitly disables expiry (redis maxmemory only)\n # key_prefix: \"\" # default: hash of the database identity\n\n# ============================================================================\n# Run Events Configuration\n# ============================================================================\n# Storage backend for run events (messages + execution traces).\n#\n# backend: memory -- No persistence, data lost on restart (default)\n# backend: db -- SQL database via ORM, full query capability (production)\n# backend: jsonl -- Append-only JSONL files (lightweight single-node persistence)\n#\n# run_events:\n# backend: memory\n# max_trace_content: 10240 # Truncation threshold for trace content (db backend, bytes)\n# track_token_usage: true # Accumulate token counts to RunRow\nrun_events:\n backend: memory\n max_trace_content: 10240\n track_token_usage: true\n\n# ============================================================================\n# Agent Storage Configuration\n# ============================================================================\n# Where custom agent DEFINITIONS (config.yaml + SOUL.md) are stored. This is\n# separate from `database` (run/thread/event data) and from agent memory.\n# Restart-required (the backend is captured at Gateway lifespan startup).\n#\n# backend: file -- Per-user files under {base_dir}/users/{uid}/agents/ (default).\n# Single-node only: an agent created on one node is invisible\n# to other nodes without a shared mount.\n# backend: db -- A row per agent in the shared SQL persistence layer, so every\n# node sees the same agents. Requires database.backend to be\n# 'sqlite' or 'postgres' (rejected at startup on 'memory').\n#\n# Switching an existing install to 'db'? Import the on-disk agents once with:\n# python backend/scripts/migrate_agents_to_db.py # --dry-run to preview\n# The source files are left in place, so reverting to 'file' is a clean rollback.\n#\n# agent_storage:\n# backend: file\nagent_storage:\n backend: file\n\n# ============================================================================\n# Scheduled Tasks Configuration\n# ============================================================================\n# Background scheduler for one-time and recurring (cron) agent runs.\n# All fields are restart-required (captured at Gateway lifespan startup).\n#\n# Multi-worker note: the scheduler runs once per uvicorn worker. SQLite silently\n# ignores row-level locks, so multiple workers can double-fire the same task.\n# For multi-worker deployments (GATEWAY_WORKERS > 1), use the Postgres database\n# backend, where FOR UPDATE SKIP LOCKED serializes claims correctly.\n#\n# scheduler:\n# enabled: false # Master switch for the background poller\n# poll_interval_seconds: 5 # How often to scan for due tasks\n# lease_seconds: 120 # Claim lease; a crashed process's task becomes reclaimable after this\n# max_concurrent_runs: 3 # Global cap on active scheduled runs; each poll claims only into the remaining budget\n# min_once_delay_seconds: 60 # Minimum future offset for one-time tasks at creation time\nscheduler:\n enabled: false\n poll_interval_seconds: 5\n lease_seconds: 120\n max_concurrent_runs: 3\n min_once_delay_seconds: 60\n\n# ============================================================================\n# Run Ownership Configuration\n# ============================================================================\n# Controls cross-process run ownership for multi-worker deployments.\n# When GATEWAY_WORKERS > 1, each worker claims runs with a lease; the heartbeat\n# renews leases, and reconciliation recovers orphaned runs from crashed workers.\n#\n# CLOCK-SYNC REQUIREMENT (multi-worker only): reconciliation compares another\n# worker's UTC lease timestamp against this worker's datetime.now(UTC). Worker\n# clocks MUST be synced (NTP / chrony / systemd-timesyncd — default on K8s and\n# cloud VMs) within a few seconds. grace_seconds is the skew budget; worst case\n# (owning worker's heartbeat just about to fire), a peer whose clock is more\n# than grace_seconds ahead can mis-reclaim a still-live run as an orphan. Raise\n# grace_seconds if your environment cannot keep clocks within a few seconds;\n# the trade-off is longer recovery latency for genuinely dead workers\n# (lease_seconds + grace_seconds from last heartbeat to reclaim).\n#\n# The owning worker itself does NOT use grace_seconds as extra execution time.\n# If renewal cannot be confirmed before lease_seconds expires, it cancels the\n# local run and suppresses durable finalization; grace_seconds only delays when\n# a peer may reclaim the expired row.\n\nrun_ownership:\n lease_seconds: 30 # Seconds before a run lease expires if not renewed.\n # Heartbeat renews every lease_seconds / 3.\n grace_seconds: 10 # Extra seconds past expiry before reclaiming an orphaned run.\n # Also the cross-worker clock-skew budget — see note above.\n heartbeat_enabled: false # Set to true for GATEWAY_WORKERS > 1\n\n# ============================================================================\n# Stream Bridge Configuration\n# ============================================================================\n# The stream bridge carries live agent events from gateway workers to SSE\n# clients. Docker Compose sets DEER_FLOW_STREAM_BRIDGE_REDIS_URL automatically,\n# so Docker deployments use Redis Streams even if this section is omitted.\n#\n# The redis bridge requires the optional `redis` extra. It is auto-detected from\n# this section on `make dev`, and always installed in the Docker image. To install\n# it manually: cd backend && uv sync --all-packages --extra redis\n#\n# stream_bridge:\n# type: memory # single-process only\n# queue_maxsize: 256 # data events retained per run. A reconnect\n# # older than this window receives SSE `gap`.\n#\n# stream_bridge:\n# type: redis # recommended for Docker / multi-worker gateway\n# redis_url: redis://redis:6379/0\n# queue_maxsize: 256 # data events retained per run (Redis MAXLEN).\n# # Older valid cursors receive SSE `gap`.\n# stream_ttl_seconds: 86400 # rolling TTL for retained stream buffers.\n# # Refreshed on each publish/publish_end; set 0\n# # to disable. This is not a run timeout.\n# recovered_stream_cleanup_delay_seconds: 60 # seconds to wait after\n# # publishing END for a recovered orphan run\n# # before deleting the stream key.\n# max_connections: 100 # optional pool ceiling. Each live SSE client\n# # holds one connection blocked in XREAD ... BLOCK\n# # for up to heartbeat_interval (15s), so hundreds\n# # of concurrent clients open hundreds of\n# # connections. Unset = redis-py default (unbounded).\n#\n# NOTE: the redis bridge is fail-hard in v1. Redis.from_url is lazy, so a down\n# Redis does not block gateway startup, but the first publish/xread raises. A\n# mid-run Redis outage fails the active run — there is no automatic retry/backoff\n# or fallback to the in-memory bridge. Run Redis with HA / a restart policy.\n\n# ============================================================================\n# User-Owned IM Channel Connections\n# ============================================================================\n# Lets logged-in users connect their own IM accounts from the DeerFlow frontend\n# while reusing the existing `channels` runtime configuration below.\n#\n# Security notes:\n# - No public IP, OAuth callback URL, or provider webhook is required.\n# - Provider bot/app credentials stay under `channels.*`.\n# - `channel_connections` stores per-user bindings and one-time connect codes.\n# - Telegram uses a deep link when `bot_username` is configured.\n# - Slack, Discord, Feishu, DingTalk, WeChat, and WeCom use `/connect `\n# through the already-running bot/app.\n#\n# channel_connections:\n# enabled: false\n# # Security: keep this enabled unless you intentionally want legacy open-bot behavior.\n# # Disabling it lets unbound external IM users create DeerFlow threads/runs.\n# require_bound_identity: true\n#\n# telegram:\n# enabled: false\n# bot_username: $TELEGRAM_BOT_USERNAME\n#\n# slack:\n# enabled: false\n#\n# discord:\n# enabled: false\n#\n# feishu:\n# enabled: false\n#\n# dingtalk:\n# enabled: false\n#\n# wechat:\n# enabled: false\n#\n# wecom:\n# enabled: false\n\n# ============================================================================\n# IM Channels Configuration\n# ============================================================================\n# Connect DeerFlow to external messaging platforms.\n# All channels use outbound connections (WebSocket or polling) — no public IP required.\n\n# channels:\n# # LangGraph-compatible Gateway API base URL for thread/message management (default: http://localhost:8001/api)\n# # For Docker deployments, use the Docker service name instead of localhost:\n# # langgraph_url: http://gateway:8001/api\n# # gateway_url: http://gateway:8001\n# langgraph_url: http://localhost:8001/api\n# # Gateway API URL for auxiliary queries like /models, /memory (default: http://localhost:8001)\n# gateway_url: http://localhost:8001\n# #\n# # Docker Compose note:\n# # If channels run inside the gateway container, use container DNS names instead\n# # of localhost, for example:\n# # langgraph_url: http://gateway:8001/api\n# # gateway_url: http://gateway:8001\n# # You can also set DEER_FLOW_CHANNELS_LANGGRAPH_URL / DEER_FLOW_CHANNELS_GATEWAY_URL.\n#\n# # Optional: default mobile/session settings for all IM channels\n# session:\n# assistant_id: lead_agent # or a custom agent name; custom agents route via lead_agent + agent_name\n# config:\n# recursion_limit: 100\n# context:\n# thinking_enabled: true\n# is_plan_mode: false\n# subagent_enabled: false\n#\n# feishu:\n# enabled: false\n# app_id: $FEISHU_APP_ID\n# app_secret: $FEISHU_APP_SECRET\n# # domain: https://open.feishu.cn # China (default)\n# # domain: https://open.larksuite.com # International\n#\n# slack:\n# enabled: false\n# bot_token: $SLACK_BOT_TOKEN # xoxb-...\n# app_token: $SLACK_APP_TOKEN # xapp-... (Socket Mode)\n# allowed_users: [] # empty = allow all; can also be a single Slack user ID string, e.g. U123456, but list form is recommended\n#\n# telegram:\n# enabled: false\n# bot_token: $TELEGRAM_BOT_TOKEN\n# allowed_users: [] # empty = allow all\n# rich_messages: false # Bot API 10.1 Rich Messages for final Markdown responses\n#\n# wechat:\n# enabled: false\n# bot_token: $WECHAT_BOT_TOKEN\n# ilink_bot_id: $WECHAT_ILINK_BOT_ID\n# # Optional: allow first-time QR bootstrap when bot_token is absent\n# qrcode_login_enabled: true\n# # Optional: sent as iLink-App-Id header when provided\n# ilink_app_id: \"\"\n# # Optional: sent as SKRouteTag header when provided\n# route_tag: \"\"\n# allowed_users: [] # empty = allow all\n# # Optional: timing values must be positive finite seconds\n# polling_timeout: 35\n# polling_retry_delay: 5\n# # QR poll interval when qrcode_login_enabled is true\n# qrcode_poll_interval: 2\n# # QR bootstrap timeout\n# qrcode_poll_timeout: 180\n# # Optional: persist getupdates cursor under the gateway container volume\n# state_dir: ./.deer-flow/wechat/state\n# # Optional: max inbound image size in bytes before skipping download\n# max_inbound_image_bytes: 20971520\n# # Optional: max outbound image size in bytes before skipping upload\n# max_outbound_image_bytes: 20971520\n# # Optional: max inbound file size in bytes before skipping download\n# max_inbound_file_bytes: 52428800\n# # Optional: max outbound file size in bytes before skipping upload\n# max_outbound_file_bytes: 52428800\n# # Optional: allowed file extensions for regular file receive/send\n# allowed_file_extensions: [\".txt\", \".md\", \".pdf\", \".csv\", \".json\", \".yaml\", \".yml\", \".xml\", \".html\", \".log\", \".zip\", \".doc\", \".docx\", \".xls\", \".xlsx\", \".ppt\", \".pptx\", \".rtf\"]\n#\n# # Optional: channel-level session overrides\n# session:\n# assistant_id: mobile-agent # custom agent names are supported here too\n# context:\n# thinking_enabled: false\n#\n# # Optional: per-user overrides by user_id\n# users:\n# \"123456789\":\n# assistant_id: vip-agent\n# config:\n# recursion_limit: 150\n# context:\n# thinking_enabled: true\n# subagent_enabled: true\n# wecom:\n# enabled: false\n# bot_id: $WECOM_BOT_ID\n# bot_secret: $WECOM_BOT_SECRET\n#\n# dingtalk:\n# enabled: false\n# client_id: $DINGTALK_CLIENT_ID\n# client_secret: $DINGTALK_CLIENT_SECRET\n# allowed_users: [] # empty = allow all\n# card_template_id: \"\" # Optional: AI Card template ID for streaming updates\n#\n# discord:\n# enabled: false\n# bot_token: $DISCORD_BOT_TOKEN\n# allowed_guilds: [] # empty = allow all guilds; can also be a single guild ID\n# mention_only: false # If true, only respond when the bot is mentioned\n# allowed_channels: [] # Optional: channel IDs exempt from mention_only (bot responds without mention)\n# thread_mode: false # If true, group a channel conversation into a thread\n\n# ============================================================================\n# Guardrails Configuration\n# ============================================================================\n# Optional pre-execution authorization for tool calls.\n# When enabled, every tool call passes through the configured provider\n# before execution. Three options: built-in allowlist, OAP policy provider,\n# or custom provider. See backend/docs/GUARDRAILS.md for full documentation.\n#\n# Providers are loaded by class path via resolve_variable (same as models/tools).\n\n# --- Option 1: Built-in AllowlistProvider (zero external deps) ---\n# guardrails:\n# enabled: true\n# provider:\n# use: deerflow.guardrails.builtin:AllowlistProvider\n# config:\n# denied_tools: [\"bash\", \"write_file\"]\n\n# --- Option 2: OAP passport provider (open standard, any implementation) ---\n# The Open Agent Passport (OAP) spec defines passport format and decision codes.\n# Any OAP-compliant provider works. Example using APort (reference implementation):\n# pip install aport-agent-guardrails && aport setup --framework deerflow\n# guardrails:\n# enabled: true\n# provider:\n# use: aport_guardrails.providers.generic:OAPGuardrailProvider\n\n# --- Option 3: Custom provider (any class with evaluate/aevaluate methods) ---\n# guardrails:\n# enabled: true\n# provider:\n# use: my_package:MyGuardrailProvider\n# config:\n# key: value\n\n# ============================================================================\n# Authorization Configuration\n# ============================================================================\n# Fine-grained resource authorization (RBAC and beyond). Disabled by default;\n# every authenticated user has access to all resources.\n# See RFC: https://github.com/bytedance/deer-flow/issues/4063\n#\n# authorization:\n# enabled: true\n# fail_closed: true # block on provider error / unresolved identity\n# default_role: user # applied when user_role is None; built-in RBAC requires this role below\n# provider:\n# use: deerflow.authz.rbac:RbacAuthorizationProvider\n# config:\n# # A known role with no policy for a resource is unrestricted for it.\n# # Define both `tools` and `routes` wherever access should be constrained.\n# roles:\n# admin:\n# tools: {allow: \"*\"}\n# routes: {allow: \"*\"}\n# models: {allow: \"*\"}\n# user:\n# tools: {allow: \"*\", deny: [\"update_agent\"]}\n# routes: {allow: \"*\"}\n# models: {allow: \"*\"}\n# guest:\n# tools: {allow: [\"web_search\", \"read_file\"]}\n# routes: {allow: [\"threads:read\", \"runs:read\"]}\n# models: {allow: [\"gpt-4o-mini\"]}\nauthorization:\n enabled: false\n\n# ============================================================================\n# Circuit Breaker Configuration\n# ============================================================================\n# Circuit breaker for LLM calls prevents repeated requests to a failing provider.\n# When the failure threshold is reached, subsequent calls fast-fail until recovery.\n#\n# This is useful for:\n# - Avoiding rate-limit bans during provider outages\n# - Reducing resource exhaustion from retry loops\n# - Gracefully degrading when LLM services are unavailable\n\n# circuit_breaker:\n# # Number of consecutive failures before opening the circuit (default: 5)\n# failure_threshold: 5\n# # Time in seconds before attempting to recover (default: 60)\n# recovery_timeout_sec: 60\n\n# ============================================================================\n# LLM Call Concurrency Configuration\n# ============================================================================\n# Cap the number of concurrently in-flight LLM calls process-wide. A provider\n# burst-rate (limit_burst_rate) error fires on the *slope* of the request rate,\n# not on a static quota - so the morning peak (e.g. 08:30) ramping from ~0 to\n# full throttle in seconds gets rejected even when total RPM is within budget.\n# Capping concurrency caps that slope. Retries alone make it worse (they add\n# demand inside the very burst being throttled); pair this cap with the\n# decorrelated-jitter backoff already built into the LLM error-handling\n# middleware, and ideally an nginx `limit_req` at the ingress.\n#\n# 0 disables the cap (default) - existing deployments see no behavior change.\n#\n# Per-process, not per-cluster: the cap bounds in-flight LLM calls within ONE\n# gateway process. With GATEWAY_WORKERS > 1 the aggregate cap across the\n# deployment is effectively `max_concurrent_calls * GATEWAY_WORKERS`, and a\n# multi-node rollout multiplies it further - size the per-process value with\n# that in mind (and pair it with an nginx `limit_req` at the ingress for a\n# true cluster-wide slope cap).\n#\n# Startup-only: the cap is captured at the first LLM run and frozen for the\n# process lifetime. Editing `max_concurrent_calls` here takes effect only after\n# a gateway restart; the other `llm_call.*` knobs below remain hot-reloadable.\n# Freezing the cap avoids the downscale and config-freshness races that a\n# runtime-mutable, process-wide/cross-loop limiter would otherwise hit (a\n# lowered cap could keep admitting queued waiters, or a stale config snapshot\n# constructed after a fresher one could restore a higher cap).\n\n# llm_call:\n# # Max concurrently in-flight LLM calls across the whole process (default: 0 = disabled)\n# max_concurrent_calls: 0\n# # Max LLM call attempts for retriable transient errors, 1 = no retry (default: 3)\n# retry_max_attempts: 3\n# # Base delay (ms) for the decorrelated-jitter retry backoff (default: 1000)\n# retry_base_delay_ms: 1000\n# # Hard cap (ms) on any single retry backoff delay (default: 8000)\n# retry_cap_delay_ms: 8000\n# # Backoff base (ms) used ONLY for burst-rate (limit_burst_rate) 429s - higher\n# # than retry_base_delay_ms so the single burst retry lands after the throttle\n# # window subsides. Ignored when the provider sends Retry-After (default: 5000)\n# burst_retry_base_delay_ms: 5000\n\n# ============================================================================\n# SSO / OIDC Authentication (optional)\n# ============================================================================\n# Enable SSO login via any OIDC-compatible provider (Keycloak, Google, Azure AD, Okta, etc.).\n# When enabled, the login page will show SSO buttons alongside the standard email/password form.\n#\n# Provider configuration:\n# - issuer: The OIDC issuer URL (e.g. https://keycloak.example.com/realms/deerflow)\n# - client_id: OAuth2 client ID from the provider\n# - client_secret: OAuth2 client secret ($ENV_VAR references supported)\n# - redirect_uri: Callback URL. In production, this must match what you configure in\n# the provider. Defaults to a self-derived URL in development.\n#\n# Keycloak setup:\n# 1. Create a client with type \"confidential\" and Standard Flow enabled\n# 2. Add Valid Redirect URI: http://localhost:8001/api/v1/auth/callback/keycloak\n# 3. Add Web Origin: http://localhost:8001 (or your frontend origin)\n\n# ============================================================================\n# Local (email/password) authentication\n# ============================================================================\n# Self-registration via POST /api/v1/auth/register is open by default: anyone\n# who can reach the Gateway may create a regular user account.\n#\n# The OIDC provisioning policy below (allowed_email_domains, require_verified_email,\n# auto_create_users) is enforced only in the SSO callback. It does NOT constrain local\n# registration, so on an SSO-provisioned deployment leaving this open lets a visitor\n# create an account outside that policy. Set allow_registration: false there.\n#\n# The first admin account is always created through /initialize regardless of this\n# setting, so turning it off cannot lock you out of a fresh install.\n\n# auth:\n# local:\n# allow_registration: false\n#\n# oidc:\n# enabled: true\n# # Base URL of the frontend, used for redirects after SSO callback.\n# # In production behind a reverse proxy, set this to the public frontend URL\n# # (e.g. https://deerflow.example.com). In development, leave unset.\n# # frontend_base_url: http://localhost:3000\n# providers:\n# keycloak:\n# display_name: Keycloak\n# issuer: https://keycloak.example.com/realms/deerflow\n# client_id: deerflow\n# client_secret: $KEYCLOAK_CLIENT_SECRET\n# # Optional: explicitly set the callback URL.\n# # redirect_uri: https://deerflow.example.com/api/v1/auth/callback/keycloak\n# scopes:\n# - openid\n# - email\n# - profile\n# token_endpoint_auth_method: client_secret_post\n#\n# # User provisioning settings (safe defaults shown below):\n# auto_create_users: true # Auto-create DeerFlow account on first SSO login\n# require_verified_email: true # Reject SSO logins without verified email\n# # allowed_email_domains: # Restrict to specific email domains\n# # - example.com\n# admin_emails: [] # Auto-grant admin role to these emails\n#\n# # Security features (enabled by default):\n# pkce_enabled: true # PKCE (S256) for authorization code flow\n# nonce_enabled: true # Nonce validation in ID tokens\n"}}, "backend/tests/test_patched_deepseek.py": {"99c926b7bbcd0570870bc24ceb13ab934935f49c": {"t": 1786068358.032632, "n": 6941, "h": "3eab6591094a5680", "c": "\"\"\"Tests for deerflow.models.patched_deepseek.PatchedChatDeepSeek.\n\nCovers:\n- LangChain serialization protocol: is_lc_serializable, lc_secrets, to_json\n- reasoning_content restoration in _get_request_payload (single and multi-turn)\n- Positional fallback when message counts differ\n- No-op when no reasoning_content present\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom unittest.mock import MagicMock, patch\n\nfrom langchain_core.messages import AIMessage, HumanMessage\n\n\ndef _make_model(**kwargs):\n from deerflow.models.patched_deepseek import PatchedChatDeepSeek\n\n return PatchedChatDeepSeek(\n model=\"deepseek-v4-pro\",\n api_key=\"test-key\",\n **kwargs,\n )\n\n\n# ---------------------------------------------------------------------------\n# Serialization protocol\n# ---------------------------------------------------------------------------\n\n\ndef test_is_lc_serializable_returns_true():\n from deerflow.models.patched_deepseek import PatchedChatDeepSeek\n\n assert PatchedChatDeepSeek.is_lc_serializable() is True\n\n\ndef test_lc_secrets_contains_api_key_mapping():\n model = _make_model()\n secrets = model.lc_secrets\n assert \"api_key\" in secrets\n assert secrets[\"api_key\"] == \"DEEPSEEK_API_KEY\"\n assert \"openai_api_key\" in secrets\n\n\ndef test_to_json_produces_constructor_type():\n model = _make_model()\n result = model.to_json()\n assert result[\"type\"] == \"constructor\"\n assert \"kwargs\" in result\n\n\ndef test_to_json_kwargs_contains_model():\n model = _make_model()\n result = model.to_json()\n assert result[\"kwargs\"][\"model_name\"] == \"deepseek-v4-pro\"\n assert result[\"kwargs\"][\"api_base\"] == \"https://api.deepseek.com/v1\"\n\n\ndef test_to_json_kwargs_contains_custom_api_base():\n model = _make_model(api_base=\"https://ark.cn-beijing.volces.com/api/v3\")\n result = model.to_json()\n assert result[\"kwargs\"][\"api_base\"] == \"https://ark.cn-beijing.volces.com/api/v3\"\n\n\ndef test_to_json_api_key_is_masked():\n \"\"\"api_key must not appear as plain text in the serialized output.\"\"\"\n model = _make_model()\n result = model.to_json()\n api_key_value = result[\"kwargs\"].get(\"api_key\") or result[\"kwargs\"].get(\"openai_api_key\")\n assert api_key_value is None or isinstance(api_key_value, dict), f\"API key must not be plain text, got: {api_key_value!r}\"\n\n\n# ---------------------------------------------------------------------------\n# reasoning_content preservation in _get_request_payload\n# ---------------------------------------------------------------------------\n\n\ndef _make_payload_message(role: str, content: str | None = None, tool_calls: list | None = None) -> dict:\n msg: dict = {\"role\": role, \"content\": content}\n if tool_calls is not None:\n msg[\"tool_calls\"] = tool_calls\n return msg\n\n\ndef test_reasoning_content_injected_into_assistant_message():\n \"\"\"reasoning_content from additional_kwargs is restored in the payload.\"\"\"\n model = _make_model()\n\n human = HumanMessage(content=\"What is 2+2?\")\n ai = AIMessage(\n content=\"4\",\n additional_kwargs={\"reasoning_content\": \"Let me think: 2+2=4\"},\n )\n\n base_payload = {\n \"messages\": [\n _make_payload_message(\"user\", \"What is 2+2?\"),\n _make_payload_message(\"assistant\", \"4\"),\n ]\n }\n\n with patch.object(type(model).__bases__[0], \"_get_request_payload\", return_value=base_payload):\n with patch.object(model, \"_convert_input\") as mock_convert:\n mock_convert.return_value = MagicMock(to_messages=lambda: [human, ai])\n payload = model._get_request_payload([human, ai])\n\n assistant_msg = next(m for m in payload[\"messages\"] if m[\"role\"] == \"assistant\")\n assert assistant_msg[\"reasoning_content\"] == \"Let me think: 2+2=4\"\n\n\ndef test_no_reasoning_content_is_noop():\n \"\"\"Messages without reasoning_content are left unchanged.\"\"\"\n model = _make_model()\n\n human = HumanMessage(content=\"hello\")\n ai = AIMessage(content=\"hi\", additional_kwargs={})\n\n base_payload = {\n \"messages\": [\n _make_payload_message(\"user\", \"hello\"),\n _make_payload_message(\"assistant\", \"hi\"),\n ]\n }\n\n with patch.object(type(model).__bases__[0], \"_get_request_payload\", return_value=base_payload):\n with patch.object(model, \"_convert_input\") as mock_convert:\n mock_convert.return_value = MagicMock(to_messages=lambda: [human, ai])\n payload = model._get_request_payload([human, ai])\n\n assistant_msg = next(m for m in payload[\"messages\"] if m[\"role\"] == \"assistant\")\n assert \"reasoning_content\" not in assistant_msg\n\n\ndef test_reasoning_content_multi_turn():\n \"\"\"All assistant turns each get their own reasoning_content.\"\"\"\n model = _make_model()\n\n human1 = HumanMessage(content=\"Step 1?\")\n ai1 = AIMessage(content=\"A1\", additional_kwargs={\"reasoning_content\": \"Thought1\"})\n human2 = HumanMessage(content=\"Step 2?\")\n ai2 = AIMessage(content=\"A2\", additional_kwargs={\"reasoning_content\": \"Thought2\"})\n\n base_payload = {\n \"messages\": [\n _make_payload_message(\"user\", \"Step 1?\"),\n _make_payload_message(\"assistant\", \"A1\"),\n _make_payload_message(\"user\", \"Step 2?\"),\n _make_payload_message(\"assistant\", \"A2\"),\n ]\n }\n\n with patch.object(type(model).__bases__[0], \"_get_request_payload\", return_value=base_payload):\n with patch.object(model, \"_convert_input\") as mock_convert:\n mock_convert.return_value = MagicMock(to_messages=lambda: [human1, ai1, human2, ai2])\n payload = model._get_request_payload([human1, ai1, human2, ai2])\n\n assistant_msgs = [m for m in payload[\"messages\"] if m[\"role\"] == \"assistant\"]\n assert assistant_msgs[0][\"reasoning_content\"] == \"Thought1\"\n assert assistant_msgs[1][\"reasoning_content\"] == \"Thought2\"\n\n\ndef test_positional_fallback_when_count_differs():\n \"\"\"Falls back to positional matching when payload/original message counts differ.\"\"\"\n model = _make_model()\n\n human = HumanMessage(content=\"hi\")\n ai = AIMessage(content=\"hello\", additional_kwargs={\"reasoning_content\": \"My reasoning\"})\n\n # Simulate count mismatch: payload has 3 messages, original has 2\n extra_system = _make_payload_message(\"system\", \"You are helpful.\")\n base_payload = {\n \"messages\": [\n extra_system,\n _make_payload_message(\"user\", \"hi\"),\n _make_payload_message(\"assistant\", \"hello\"),\n ]\n }\n\n with patch.object(type(model).__bases__[0], \"_get_request_payload\", return_value=base_payload):\n with patch.object(model, \"_convert_input\") as mock_convert:\n mock_convert.return_value = MagicMock(to_messages=lambda: [human, ai])\n payload = model._get_request_payload([human, ai])\n\n assistant_msg = next(m for m in payload[\"messages\"] if m[\"role\"] == \"assistant\")\n assert assistant_msg[\"reasoning_content\"] == \"My reasoning\"\n"}}}}