1 line
55 KiB
JSON
1 line
55 KiB
JSON
{"repo": "0xsmw/basecamp-bench", "files": {"basecamp_bench/adapters.py": {"4ddd161ff78ea1467fb4f9d00d89960b0742595c": {"t": 1785595958.8469102, "n": 53590, "h": "4f0e6ff77c22dab0", "c": "\"\"\"Agent-harness adapter layer for Basecamp Bench.\n\nIsolates CLI-specific command construction, environment allowlisting, and\noutput parsing behind a small registry of :class:`Harness` adapters. This\nmodule is intentionally free of runner pipeline imports (no ``run_bench``).\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nimport shutil\nimport stat\nimport sys\nfrom abc import ABC, abstractmethod\nfrom collections.abc import Callable, Iterable, Iterator, Mapping\nfrom contextlib import contextmanager\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom typing import Any, ClassVar, Literal\n\n__all__ = [\n \"ModelSpec\",\n \"Usage\",\n \"ParsedOutput\",\n \"AgentJob\",\n \"Harness\",\n \"CodexHarness\",\n \"ClaudeHarness\",\n \"GrokHarness\",\n \"PiHarness\",\n \"AgyHarness\",\n \"register_harness\",\n \"get_harness\",\n \"registered_harnesses\",\n \"is_retained_env_name\",\n \"RETAINED_ENV_NAMES\",\n \"DEFAULT_GROK_BINARY\",\n \"PI_MODEL_ALIASES\",\n \"AGY_MODEL_ALIASES\",\n]\n\n\n# =============================================================================\n# Immutable value types\n# =============================================================================\n\n\n@dataclass(frozen=True)\nclass ModelSpec:\n \"\"\"Harness-native model identifier and reasoning effort.\"\"\"\n\n model: str\n effort: str\n\n\n@dataclass(frozen=True)\nclass Usage:\n \"\"\"Normalized, disjoint token buckets.\n\n ``input_tokens`` excludes cached reads and cache writes. Adapters normalize\n each CLI's native semantics into these four buckets.\n \"\"\"\n\n input_tokens: int = 0\n cached_input_tokens: int = 0\n cache_write_tokens: int = 0\n output_tokens: int = 0\n\n def add(self, other: Usage) -> Usage:\n \"\"\"Return a new :class:`Usage` with bucket-wise sums.\"\"\"\n return Usage(\n input_tokens=self.input_tokens + other.input_tokens,\n cached_input_tokens=self.cached_input_tokens + other.cached_input_tokens,\n cache_write_tokens=self.cache_write_tokens + other.cache_write_tokens,\n output_tokens=self.output_tokens + other.output_tokens,\n )\n\n def total(self) -> int:\n \"\"\"Sum of all four token buckets.\"\"\"\n return (\n self.input_tokens\n + self.cached_input_tokens\n + self.cache_write_tokens\n + self.output_tokens\n )\n\n\n@dataclass(frozen=True)\nclass ParsedOutput:\n \"\"\"Trustworthy fields recovered from a harness CLI's stdout/log text.\n\n Never contains secret values or global log paths — only usage, cost,\n final message text, and session identifiers when confidently parsed.\n \"\"\"\n\n usage: Usage | None = None\n reported_cost_usd: float | None = None\n last_message: str | None = None\n session_id: str | None = None\n\n\n@dataclass(frozen=True)\nclass AgentJob:\n \"\"\"One isolated implement or evaluate invocation for a harness adapter.\"\"\"\n\n kind: Literal[\"implement\", \"evaluate\"]\n harness: str\n model: ModelSpec\n workdir: Path\n prompt_path: Path\n log_path: Path\n last_message_path: Path\n evidence_dirs: tuple[Path, ...] = ()\n sandbox_mode: str = \"workspace-write\"\n\n def __post_init__(self) -> None:\n if self.kind not in (\"implement\", \"evaluate\"):\n raise ValueError(f\"AgentJob.kind must be 'implement' or 'evaluate', got {self.kind!r}\")\n\n\n# =============================================================================\n# Environment allowlist (names only — values never logged by this module)\n# =============================================================================\n\n# Exact names retained by :meth:`Harness.prepare_env`. Exported for tests.\nRETAINED_ENV_NAMES: frozenset[str] = frozenset(\n {\n # Runtime / paths\n \"PATH\",\n \"HOME\",\n \"USER\",\n \"LOGNAME\",\n \"SHELL\",\n \"TERM\",\n \"TMPDIR\",\n \"TEMP\",\n \"TMP\",\n # Locale\n \"LANG\",\n \"LANGUAGE\",\n \"LC_ALL\",\n \"LC_CTYPE\",\n \"LC_MESSAGES\",\n \"LC_COLLATE\",\n \"LC_MONETARY\",\n \"LC_NUMERIC\",\n \"LC_TIME\",\n \"TZ\",\n # SSL / certificates\n \"SSL_CERT_FILE\",\n \"SSL_CERT_DIR\",\n \"REQUESTS_CA_BUNDLE\",\n \"CURL_CA_BUNDLE\",\n \"NODE_EXTRA_CA_CERTS\",\n # Proxy (common cases, both casings)\n \"HTTP_PROXY\",\n \"HTTPS_PROXY\",\n \"ALL_PROXY\",\n \"NO_PROXY\",\n \"http_proxy\",\n \"https_proxy\",\n \"all_proxy\",\n \"no_proxy\",\n # OpenAI / Codex\n \"OPENAI_API_KEY\",\n \"OPENAI_BASE_URL\",\n \"OPENAI_API_BASE\",\n \"OPENAI_ORG_ID\",\n \"OPENAI_ORGANIZATION\",\n \"OPENAI_PROJECT\",\n \"CODEX_API_KEY\",\n # Anthropic / Claude\n \"ANTHROPIC_API_KEY\",\n \"ANTHROPIC_AUTH_TOKEN\",\n \"ANTHROPIC_BASE_URL\",\n \"ANTHROPIC_API_BASE\",\n \"CLAUDE_API_KEY\",\n \"CLAUDE_CODE_API_KEY\",\n \"CLAUDE_CODE_OAUTH_TOKEN\",\n # Grok / xAI\n \"XAI_API_KEY\",\n \"XAI_BASE_URL\",\n \"GROK_API_KEY\",\n \"GROK_HOME\",\n # Pi / OpenRouter\n \"OPENROUTER_API_KEY\",\n }\n)\n\n_RETAINED_ENV_PREFIXES: tuple[str, ...] = (\"LC_\",)\n\n\ndef is_retained_env_name(name: str) -> bool:\n \"\"\"Return True if *name* is on the portable environment allowlist.\n\n Matching is by exact name (see :data:`RETAINED_ENV_NAMES`) or by the\n ``LC_`` locale prefix. Values are never inspected.\n \"\"\"\n if name in RETAINED_ENV_NAMES:\n return True\n return any(name.startswith(prefix) for prefix in _RETAINED_ENV_PREFIXES)\n\n\n# =============================================================================\n# JSON helpers\n# =============================================================================\n\n\ndef _iter_json_objects(text: str) -> Iterable[dict[str, Any]]:\n \"\"\"Yield JSON objects from whole-body JSON or JSONL; noise-tolerant.\"\"\"\n body = text.strip()\n if body.startswith(\"{\"):\n try:\n obj = json.loads(body)\n if isinstance(obj, dict):\n yield obj\n return\n except json.JSONDecodeError:\n pass\n for line in text.splitlines():\n line = line.strip()\n if not line.startswith(\"{\"):\n continue\n try:\n obj = json.loads(line)\n except json.JSONDecodeError:\n continue\n if isinstance(obj, dict):\n yield obj\n\n\ndef _int_of(value: Any) -> int:\n return int(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else 0\n\n\ndef _float_of(value: Any) -> float | None:\n if isinstance(value, bool):\n return None\n if isinstance(value, (int, float)):\n return float(value)\n return None\n\n\n# =============================================================================\n# Harness interface\n# =============================================================================\n\n\nclass Harness(ABC):\n \"\"\"Adapter for one agent CLI.\n\n Subclass, set ``name``, implement :meth:`build_command`, and register with\n :func:`register_harness`. Override cwd/stdin/env/parse hooks as needed.\n \"\"\"\n\n name: ClassVar[str]\n requires_usage: ClassVar[bool] = False\n\n def __init__(self, binary: str | None = None) -> None:\n self._binary = binary\n\n def configured_binary(self) -> str:\n \"\"\"Configured binary name or path (not necessarily resolved).\"\"\"\n return self._binary if self._binary is not None else self.name\n\n def resolve_binary(self) -> str:\n \"\"\"Resolve the configured binary to an executable path.\n\n Absolute or path-like configurations are checked exactly as given and\n never silently substituted with a same-named executable from ``PATH``.\n Bare names are looked up via ``PATH``. Raises :class:`FileNotFoundError`\n with a clear message when the configured binary is missing.\n \"\"\"\n configured = self.configured_binary()\n if not configured:\n raise FileNotFoundError(f\"Harness '{self.name}': no binary configured\")\n looks_like_path = (\n os.path.isabs(configured)\n or os.sep in configured\n or (os.altsep is not None and os.altsep in configured)\n )\n if looks_like_path:\n path = Path(configured)\n if not path.is_file():\n raise FileNotFoundError(\n f\"Harness '{self.name}': configured binary not found: {configured}\"\n )\n return str(path)\n found = shutil.which(configured)\n if not found:\n raise FileNotFoundError(\n f\"Harness '{self.name}': binary '{configured}' not found on PATH\"\n )\n return found\n\n def version_command(self) -> list[str]:\n \"\"\"Minimal, side-effect-free argv that prints the CLI version.\"\"\"\n return [self.resolve_binary(), \"--version\"]\n\n @abstractmethod\n def build_command(self, job: AgentJob) -> list[str]:\n \"\"\"Build argv for a full agent run (list of strings, never a shell string).\n\n Prompt contents must not appear in argv — use stdin or a prompt file.\n \"\"\"\n\n def working_directory(self, job: AgentJob) -> Path | None:\n \"\"\"Subprocess cwd, or ``None`` when the CLI sets workspace via flags only.\n\n Default: ``job.workdir``.\n \"\"\"\n return job.workdir\n\n def stdin_for(self, job: AgentJob) -> bytes | None:\n \"\"\"Bytes to pass as stdin, or ``None`` when unused.\"\"\"\n return None\n\n @contextmanager\n def execution_context(self, job: AgentJob) -> Iterator[None]:\n \"\"\"Prepare transient per-job state, restoring it after execution.\"\"\"\n yield\n\n def prepare_env(self, base: Mapping[str, str] | None = None) -> dict[str, str]:\n \"\"\"Build a fresh environment from the portable allowlist.\n\n Copies only retained names from *base* (or ``os.environ`` when *base*\n is ``None``). Does not pass through arbitrary host variables.\n \"\"\"\n source: Mapping[str, str] = os.environ if base is None else base\n return {k: v for k, v in source.items() if is_retained_env_name(k)}\n\n def parse_output(self, job: AgentJob, stdout_text: str) -> ParsedOutput:\n \"\"\"Recover usage / cost / message / session from stdout or command log text.\n\n Best-effort and non-raising for ordinary parse failures: return whatever\n trustworthy fields were recovered. Never returns secret values or global\n log paths.\n \"\"\"\n return ParsedOutput()\n\n\n# =============================================================================\n# Registry\n# =============================================================================\n\n_HARNESS_TYPES: dict[str, type[Harness]] = {}\n\n\ndef register_harness(\n cls: type[Harness] | None = None,\n *,\n replace: bool = False,\n) -> type[Harness] | Callable[[type[Harness]], type[Harness]]:\n \"\"\"Register a :class:`Harness` subclass under its ``name`` ClassVar.\n\n May be used as ``@register_harness`` or ``@register_harness(replace=True)``.\n Rejects missing/empty names and duplicate names unless *replace* is True.\n \"\"\"\n\n def decorator(harness_cls: type[Harness]) -> type[Harness]:\n name = getattr(harness_cls, \"name\", None)\n if not isinstance(name, str) or not name.strip():\n raise ValueError(f\"{harness_cls.__name__} must define a non-empty ClassVar 'name'\")\n if name in _HARNESS_TYPES and not replace:\n raise ValueError(\n f\"Harness '{name}' is already registered; pass replace=True to override\"\n )\n _HARNESS_TYPES[name] = harness_cls\n return harness_cls\n\n if cls is not None:\n return decorator(cls)\n return decorator\n\n\ndef get_harness(name: str, *, binary: str | None = None) -> Harness:\n \"\"\"Return a new adapter instance for *name*.\n\n Raises :class:`KeyError` for unknown harness names. Optional *binary*\n overrides the adapter's default executable configuration.\n \"\"\"\n try:\n harness_cls = _HARNESS_TYPES[name]\n except KeyError as exc:\n known = \", \".join(registered_harnesses()) or \"(none)\"\n raise KeyError(f\"Unknown harness '{name}'. Registered: {known}\") from exc\n return harness_cls(binary=binary)\n\n\ndef registered_harnesses() -> list[str]:\n \"\"\"Sorted list of registered harness names (deterministic order).\"\"\"\n return sorted(_HARNESS_TYPES)\n\n\n# =============================================================================\n# Built-in harnesses\n# =============================================================================\n\n\n@register_harness\nclass CodexHarness(Harness):\n \"\"\"OpenAI Codex CLI — ``exec``, JSONL events, prompt on stdin.\"\"\"\n\n name = \"codex\"\n\n def working_directory(self, job: AgentJob) -> Path | None:\n # Workspace is selected with ``-C``; keep process cwd neutral.\n return None\n\n def stdin_for(self, job: AgentJob) -> bytes | None:\n return job.prompt_path.read_bytes()\n\n def build_command(self, job: AgentJob) -> list[str]:\n cmd: list[str] = [\n self.resolve_binary(),\n \"exec\",\n \"-m\",\n job.model.model,\n \"-c\",\n f\"model_reasoning_effort={job.model.effort}\",\n \"-C\",\n str(job.workdir),\n \"-s\",\n job.sandbox_mode,\n \"--skip-git-repo-check\",\n \"--json\",\n \"-o\",\n str(job.last_message_path),\n ]\n if job.sandbox_mode == \"danger-full-access\":\n cmd.append(\"--dangerously-bypass-approvals-and-sandbox\")\n # Evaluators may only read explicit evidence dirs (never implicit provenance).\n for evidence in job.evidence_dirs:\n cmd.extend([\"--add-dir\", str(evidence)])\n cmd.append(\"-\") # prompt on stdin\n return cmd\n\n def parse_output(self, job: AgentJob, stdout_text: str) -> ParsedOutput:\n # Codex ``input_tokens`` includes cached reads; normalize to disjoint buckets.\n # Prefer a final cumulative ``token_count`` when present; otherwise sum\n # demonstrably per-turn ``turn.completed`` usage events.\n def from_codex(data: dict[str, Any]) -> Usage:\n raw_input = _int_of(data.get(\"input_tokens\"))\n cached = _int_of(data.get(\"cached_input_tokens\"))\n cache_write = _int_of(\n data.get(\"cache_write_tokens\") or data.get(\"cache_creation_input_tokens\")\n )\n return Usage(\n input_tokens=max(raw_input - cached, 0),\n cached_input_tokens=cached,\n cache_write_tokens=cache_write,\n output_tokens=_int_of(data.get(\"output_tokens\")),\n )\n\n cumulative: Usage | None = None\n per_turn = Usage()\n saw_turn = False\n last_message: str | None = None\n session_id: str | None = None\n cost: float | None = None\n\n for obj in _iter_json_objects(stdout_text):\n if obj.get(\"type\") == \"turn.completed\" and isinstance(obj.get(\"usage\"), dict):\n per_turn = per_turn.add(from_codex(obj[\"usage\"]))\n saw_turn = True\n\n msg = obj.get(\"msg\")\n if isinstance(msg, dict) and msg.get(\"type\") == \"token_count\":\n info = msg.get(\"info\") or {}\n if isinstance(info, dict):\n total = info.get(\"total_token_usage\")\n if isinstance(total, dict):\n cumulative = from_codex(total)\n\n if isinstance(obj.get(\"session_id\"), str):\n session_id = obj[\"session_id\"]\n if isinstance(obj.get(\"sessionId\"), str):\n session_id = obj[\"sessionId\"]\n\n if isinstance(obj.get(\"last_agent_message\"), str):\n last_message = obj[\"last_agent_message\"]\n result = obj.get(\"result\")\n if isinstance(result, str):\n last_message = result\n\n reported = obj.get(\"total_cost_usd\")\n parsed_cost = _float_of(reported)\n if parsed_cost is not None:\n cost = parsed_cost\n\n usage = cumulative if cumulative is not None else (per_turn if saw_turn else None)\n return ParsedOutput(\n usage=usage,\n reported_cost_usd=cost,\n last_message=last_message,\n session_id=session_id,\n )\n\n\n@register_harness\nclass ClaudeHarness(Harness):\n \"\"\"Claude Code — headless ``-p`` with prompt on stdin, JSON result.\"\"\"\n\n name = \"claude\"\n\n def stdin_for(self, job: AgentJob) -> bytes | None:\n return job.prompt_path.read_bytes()\n\n @staticmethod\n def _sandbox_settings(job: AgentJob) -> str:\n \"\"\"Return fail-closed Bash sandbox settings scoped to this job.\"\"\"\n read_only = [job.workdir, *job.evidence_dirs]\n system_roots = (\n Path(\"/System\"),\n Path(\"/Library\"),\n Path(\"/bin\"),\n Path(\"/sbin\"),\n Path(\"/usr/bin\"),\n Path(\"/usr/sbin\"),\n Path(\"/usr/lib\"),\n Path(\"/lib\"),\n Path(\"/lib64\"),\n Path(\"/etc\"),\n Path(\"/dev\"),\n )\n for path in system_roots:\n if path.is_dir() and path not in read_only:\n read_only.append(path)\n for entry in os.environ.get(\"PATH\", \"\").split(os.pathsep):\n path = Path(entry)\n if entry and path.is_absolute() and path.is_dir() and path not in read_only:\n read_only.append(path)\n\n deny_tools: list[str] = []\n for evidence in job.evidence_dirs:\n deny_tools.extend(\n [\n f\"Write({evidence})\",\n f\"Write({evidence}/**)\",\n f\"Edit({evidence})\",\n f\"Edit({evidence}/**)\",\n ]\n )\n settings = {\n \"permissions\": {\"deny\": deny_tools},\n \"sandbox\": {\n \"enabled\": True,\n \"failIfUnavailable\": True,\n \"autoAllowBashIfSandboxed\": True,\n \"allowUnsandboxedCommands\": False,\n \"excludedCommands\": [],\n \"filesystem\": {\n \"denyRead\": [\"/**\"],\n \"allowRead\": [str(path) for path in read_only],\n \"allowWrite\": [str(job.workdir)],\n \"denyWrite\": [str(path) for path in job.evidence_dirs],\n },\n \"network\": {\n \"allowedDomains\": [\"*\"],\n \"allowAllUnixSockets\": True,\n \"allowLocalBinding\": True,\n },\n },\n }\n return json.dumps(settings, separators=(\",\", \":\"), sort_keys=True)\n\n def build_command(self, job: AgentJob) -> list[str]:\n # With no positional prompt, ``-p`` reads the prompt from stdin.\n cmd: list[str] = [\n self.resolve_binary(),\n \"-p\",\n \"--model\",\n job.model.model,\n \"--effort\",\n job.model.effort,\n \"--output-format\",\n \"json\",\n \"--no-session-persistence\",\n ]\n if job.sandbox_mode == \"danger-full-access\":\n cmd.extend([\"--permission-mode\", \"bypassPermissions\"])\n else:\n cmd.extend(\n [\n \"--settings\",\n self._sandbox_settings(job),\n \"--permission-mode\",\n \"dontAsk\",\n \"--allowedTools\",\n \"Bash,Edit,Glob,Grep,Read,Write\",\n ]\n )\n for evidence in job.evidence_dirs:\n cmd.extend([\"--add-dir\", str(evidence)])\n return cmd\n\n def parse_output(self, job: AgentJob, stdout_text: str) -> ParsedOutput:\n # Prefer summing ``modelUsage`` (per-model, includes subagents). Fall back\n # to top-level ``usage``. Buckets are already disjoint in Claude output.\n for obj in _iter_json_objects(stdout_text):\n if obj.get(\"type\") != \"result\" and \"usage\" not in obj and \"modelUsage\" not in obj:\n continue\n usage: Usage | None = None\n model_usage = obj.get(\"modelUsage\")\n if isinstance(model_usage, dict) and model_usage:\n usage = Usage()\n for mu in model_usage.values():\n if not isinstance(mu, dict):\n continue\n usage = usage.add(\n Usage(\n input_tokens=_int_of(mu.get(\"inputTokens\")),\n cached_input_tokens=_int_of(mu.get(\"cacheReadInputTokens\")),\n cache_write_tokens=_int_of(mu.get(\"cacheCreationInputTokens\")),\n output_tokens=_int_of(mu.get(\"outputTokens\")),\n )\n )\n elif isinstance(obj.get(\"usage\"), dict):\n u = obj[\"usage\"]\n usage = Usage(\n input_tokens=_int_of(u.get(\"input_tokens\")),\n cached_input_tokens=_int_of(u.get(\"cache_read_input_tokens\")),\n cache_write_tokens=_int_of(u.get(\"cache_creation_input_tokens\")),\n output_tokens=_int_of(u.get(\"output_tokens\")),\n )\n\n cost = _float_of(obj.get(\"total_cost_usd\"))\n result = obj.get(\"result\")\n last_message = result if isinstance(result, str) else None\n session_id = None\n for key in (\"session_id\", \"sessionId\"):\n val = obj.get(key)\n if isinstance(val, str):\n session_id = val\n break\n return ParsedOutput(\n usage=usage,\n reported_cost_usd=cost,\n last_message=last_message,\n session_id=session_id,\n )\n return ParsedOutput()\n\n\nDEFAULT_GROK_BINARY = \"grok\"\n\n_GROK_DISALLOWED_TOOLS = (\n \"ask_user_question\",\n \"image_edit\",\n \"image_gen\",\n \"lsp\",\n \"memory_get\",\n \"memory_search\",\n \"monitor\",\n \"scheduler_create\",\n \"scheduler_delete\",\n \"scheduler_list\",\n \"search_tool\",\n \"todo_write\",\n \"use_tool\",\n \"video_gen\",\n \"web_fetch\",\n \"web_search\",\n)\n\n\n@register_harness\nclass GrokHarness(Harness):\n \"\"\"Grok Build — ``--prompt-file`` + ``--cwd``, JSON output, scoped permissions.\"\"\"\n\n name = \"grok\"\n\n def __init__(self, binary: str | None = None) -> None:\n # Community installs resolve from PATH; local config may pin an absolute\n # binary when multiple CLIs share the same command name.\n super().__init__(binary=DEFAULT_GROK_BINARY if binary is None else binary)\n\n def working_directory(self, job: AgentJob) -> Path | None:\n # Workspace is selected with ``--cwd``; process cwd optional.\n return None\n\n def stdin_for(self, job: AgentJob) -> bytes | None:\n # Prompt is carried via ``--prompt-file`` only.\n return None\n\n def prepare_env(self, base: Mapping[str, str] | None = None) -> dict[str, str]:\n env = super().prepare_env(base)\n # Keep each benchmark submission to one model agent. Unlike\n # ``--no-subagents``, this does not remove the task-output companions\n # that Grok's background-capable terminal requires at session startup.\n env[\"GROK_SUBAGENTS\"] = \"0\"\n return env\n\n @contextmanager\n def execution_context(self, job: AgentJob) -> Iterator[None]:\n \"\"\"Install a fail-closed custom OS sandbox profile for this job.\"\"\"\n if job.sandbox_mode == \"danger-full-access\":\n yield\n return\n\n grok_dir = job.workdir / \".grok\"\n profile_path = grok_dir / \"sandbox.toml\"\n prompt_copy = grok_dir / \".basecamp-bench-prompt.md\"\n if grok_dir.is_symlink() or profile_path.is_symlink() or prompt_copy.is_symlink():\n raise ValueError(\"Grok sandbox configuration path must not be a symlink\")\n\n created_dir = not grok_dir.exists()\n if created_dir:\n grok_dir.mkdir()\n elif not grok_dir.is_dir():\n raise ValueError(\"Grok sandbox configuration parent must be a directory\")\n\n original = profile_path.read_bytes() if profile_path.exists() else None\n if original is not None and b\"[profiles.basecamp_bench]\" in original:\n raise ValueError(\"workspace already defines reserved Grok sandbox profile\")\n if prompt_copy.exists():\n raise ValueError(\"workspace contains reserved Grok prompt path\")\n\n read_only = list(job.evidence_dirs)\n if any(not path.is_dir() for path in read_only):\n raise ValueError(\"Grok evidence paths must be directories\")\n for entry in os.environ.get(\"PATH\", \"\").split(os.pathsep):\n path = Path(entry)\n if entry and path.is_absolute() and path.is_dir() and path not in read_only:\n read_only.append(path)\n profile = (\n \"\\n[profiles.basecamp_bench]\\n\"\n 'extends = \"strict\"\\n'\n \"restrict_network = false\\n\"\n f\"read_only = {json.dumps([str(path) for path in read_only])}\\n\"\n ).encode()\n prefix = b\"\" if original is None or original.endswith(b\"\\n\") else b\"\\n\"\n profile_path.write_bytes((original or b\"\") + prefix + profile)\n prompt_copy.write_bytes(job.prompt_path.read_bytes())\n try:\n yield\n finally:\n if grok_dir.is_symlink():\n return\n if profile_path.is_symlink():\n profile_path.unlink()\n if prompt_copy.is_symlink() or prompt_copy.is_file():\n prompt_copy.unlink()\n if original is None:\n if profile_path.is_file():\n profile_path.unlink()\n else:\n profile_path.write_bytes(original)\n if created_dir:\n try:\n grok_dir.rmdir()\n except OSError:\n pass\n\n def build_command(self, job: AgentJob) -> list[str]:\n cmd: list[str] = [\n self.resolve_binary(),\n \"--prompt-file\",\n str(\n job.prompt_path\n if job.sandbox_mode == \"danger-full-access\"\n else job.workdir / \".grok\" / \".basecamp-bench-prompt.md\"\n ),\n \"--cwd\",\n str(job.workdir),\n \"-m\",\n job.model.model,\n \"--reasoning-effort\",\n job.model.effort,\n \"--output-format\",\n \"json\",\n \"--no-memory\",\n \"--verbatim\",\n ]\n if job.sandbox_mode == \"danger-full-access\":\n cmd.append(\"--always-approve\")\n else:\n cmd.extend(\n [\n \"--sandbox\",\n \"basecamp_bench\",\n \"--permission-mode\",\n \"dontAsk\",\n \"--disallowed-tools\",\n # Grok 0.2.93 leaves Bash auto-backgrounding enabled while\n # every positive --tools allowlist disables background\n # support, causing session construction to fail. Start from\n # its coherent default tool graph and remove capabilities\n # the benchmark does not need. The OS sandbox and explicit\n # permission rules below remain the filesystem boundary.\n \",\".join(_GROK_DISALLOWED_TOOLS),\n ]\n )\n cmd.extend(self._permission_rules(job))\n return cmd\n\n @staticmethod\n def _permission_rules(job: AgentJob) -> list[str]:\n \"\"\"Build ``--allow`` / ``--deny`` rules from workdir and evidence_dirs only.\"\"\"\n rules: list[str] = []\n workdir = Path(job.workdir)\n allowed_roots = (workdir, *job.evidence_dirs)\n\n # Read/search across workdir and evidence; write only inside workdir.\n for root in allowed_roots:\n root_s = str(root)\n rules.extend(\n [\n \"--allow\",\n f\"Read({root_s})\",\n \"--allow\",\n f\"Read({root_s}/**)\",\n \"--allow\",\n f\"Grep({root_s}/**)\",\n ]\n )\n workdir_s = str(workdir)\n rules.extend(\n [\n \"--allow\",\n \"Bash(*)\",\n \"--allow\",\n f\"Write({workdir_s})\",\n \"--allow\",\n f\"Write({workdir_s}/**)\",\n \"--allow\",\n f\"Edit({workdir_s})\",\n \"--allow\",\n f\"Edit({workdir_s}/**)\",\n ]\n )\n # Evidence trees are read-only: deny write/edit constructed only from\n # the supplied evidence_dirs (no implicit provenance paths).\n for evidence in job.evidence_dirs:\n evidence_s = str(evidence)\n rules.extend(\n [\n \"--deny\",\n f\"Write({evidence_s})\",\n \"--deny\",\n f\"Write({evidence_s}/**)\",\n \"--deny\",\n f\"Edit({evidence_s})\",\n \"--deny\",\n f\"Edit({evidence_s}/**)\",\n ]\n )\n return rules\n\n def parse_output(self, job: AgentJob, stdout_text: str) -> ParsedOutput:\n # Prefer the explicit command log / stdout. Usage may appear inline in\n # newer JSON; otherwise, only consult the global unified log when a\n # session id was observed and matches log event ``sid`` values.\n session_id: str | None = None\n last: str | None = None\n cost: float | None = None\n usage: Usage | None = None\n\n for obj in _iter_json_objects(stdout_text):\n if isinstance(obj.get(\"text\"), str):\n last = obj[\"text\"]\n if isinstance(obj.get(\"sessionId\"), str):\n session_id = obj[\"sessionId\"]\n if isinstance(obj.get(\"session_id\"), str):\n session_id = obj[\"session_id\"]\n\n parsed_cost = _float_of(obj.get(\"total_cost_usd\") or obj.get(\"cost_usd\"))\n if parsed_cost is not None:\n cost = parsed_cost\n\n # Inline usage (when present) — treat as authoritative for this object.\n if isinstance(obj.get(\"usage\"), dict):\n inline = self._usage_from_mapping(obj[\"usage\"])\n elif any(\n key in obj\n for key in (\n \"prompt_tokens\",\n \"completion_tokens\",\n \"input_tokens\",\n \"output_tokens\",\n \"cached_prompt_tokens\",\n )\n ):\n inline = self._usage_from_mapping(obj)\n else:\n inline = None\n if inline is not None and inline.total() > 0:\n usage = inline\n\n if usage is None and session_id:\n usage = self._usage_from_global_log(session_id)\n\n return ParsedOutput(\n usage=usage,\n reported_cost_usd=cost,\n last_message=last,\n session_id=session_id,\n )\n\n @staticmethod\n def _usage_from_mapping(data: Mapping[str, Any] | None) -> Usage | None:\n if not isinstance(data, Mapping):\n return None\n # Support both Grok-native and OpenAI-ish field names.\n if \"prompt_tokens\" in data or \"cached_prompt_tokens\" in data or \"completion_tokens\" in data:\n prompt = _int_of(data.get(\"prompt_tokens\"))\n cached = _int_of(data.get(\"cached_prompt_tokens\"))\n cache_write = _int_of(\n data.get(\"cache_write_tokens\") or data.get(\"cache_creation_tokens\")\n )\n return Usage(\n input_tokens=max(prompt - cached, 0),\n cached_input_tokens=cached,\n cache_write_tokens=cache_write,\n output_tokens=_int_of(data.get(\"completion_tokens\") or data.get(\"output_tokens\")),\n )\n if \"input_tokens\" in data or \"output_tokens\" in data:\n raw_input = _int_of(data.get(\"input_tokens\"))\n cached = _int_of(data.get(\"cached_input_tokens\") or data.get(\"cache_read_input_tokens\"))\n return Usage(\n input_tokens=max(raw_input - cached, 0)\n if \"cached_input_tokens\" in data or \"cache_read_input_tokens\" in data\n else raw_input,\n cached_input_tokens=cached,\n cache_write_tokens=_int_of(\n data.get(\"cache_write_tokens\") or data.get(\"cache_creation_input_tokens\")\n ),\n output_tokens=_int_of(data.get(\"output_tokens\")),\n )\n return None\n\n @staticmethod\n def _usage_from_global_log(session_id: str) -> Usage | None:\n \"\"\"Sum incremental inference events for *session_id* only.\n\n Never returns the log path. Events without a matching ``sid`` are ignored.\n \"\"\"\n if not session_id:\n return None\n log_path = (\n Path(os.environ.get(\"GROK_HOME\", Path.home() / \".grok\")) / \"logs\" / \"unified.jsonl\"\n )\n if not log_path.is_file():\n return None\n total = Usage()\n found = False\n try:\n with log_path.open(encoding=\"utf-8\", errors=\"replace\") as fh:\n for line in fh:\n if session_id not in line or \"inference_done\" not in line:\n continue\n try:\n obj = json.loads(line)\n except json.JSONDecodeError:\n continue\n if obj.get(\"sid\") != session_id:\n continue\n ctx = obj.get(\"ctx\") or {}\n if not isinstance(ctx, dict):\n continue\n prompt = _int_of(ctx.get(\"prompt_tokens\"))\n cached = _int_of(ctx.get(\"cached_prompt_tokens\"))\n cache_write = _int_of(\n ctx.get(\"cache_write_tokens\") or ctx.get(\"cache_creation_tokens\")\n )\n total = total.add(\n Usage(\n input_tokens=max(prompt - cached, 0),\n cached_input_tokens=cached,\n cache_write_tokens=cache_write,\n output_tokens=_int_of(ctx.get(\"completion_tokens\")),\n )\n )\n found = True\n except OSError:\n return None\n return total if found else None\n\n\nPI_MODEL_ALIASES: Mapping[str, str] = {\n # Benchmark identifiers must be safe path components, while OpenRouter's\n # native model identifiers contain a provider slash.\n \"glm-5.2\": \"z-ai/glm-5.2\",\n}\n\n# Pi lanes authenticate via $OPENROUTER_API_KEY in the generated catalog. The\n# variable is only forwarded when the process launching the benchmark has it,\n# which silently breaks runs started outside an interactive shell. This file\n# is the durable fallback; the env var still wins when both are present.\nOPENROUTER_KEY_FILE: Path = Path.home() / \".config\" / \"basecamp-bench\" / \"openrouter-api-key\"\n\n\ndef _read_openrouter_key_file() -> str | None:\n flags = os.O_RDONLY | getattr(os, \"O_CLOEXEC\", 0) | getattr(os, \"O_NOFOLLOW\", 0)\n try:\n fd = os.open(OPENROUTER_KEY_FILE, flags)\n except OSError:\n return None\n try:\n info = os.fstat(fd)\n if (\n not stat.S_ISREG(info.st_mode)\n or info.st_uid != os.geteuid()\n or info.st_nlink != 1\n or stat.S_IMODE(info.st_mode) & 0o077\n ):\n return None\n with os.fdopen(fd, \"r\", encoding=\"utf-8\") as handle:\n fd = -1\n value = handle.read(16_385)\n except (OSError, UnicodeError):\n return None\n finally:\n if fd >= 0:\n os.close(fd)\n if len(value) > 16_384:\n return None\n return value.strip() or None\n\n\n@register_harness\nclass PiHarness(Harness):\n \"\"\"Pi coding agent using an isolated OpenRouter model catalog.\"\"\"\n\n name = \"pi\"\n requires_usage = True\n _PRIVATE_SUFFIX = \".pi-state\"\n\n def __init__(self, binary: str | None = None) -> None:\n super().__init__(binary=binary)\n self._active_config_dir: Path | None = None\n\n @classmethod\n def _private_root(cls, job: AgentJob) -> Path:\n return job.log_path.parent / f\".{job.log_path.name}{cls._PRIVATE_SUFFIX}\"\n\n @classmethod\n def _session_dir(cls, job: AgentJob) -> Path:\n return cls._private_root(job) / \"sessions\"\n\n @staticmethod\n def _native_model(model: str) -> str:\n try:\n return PI_MODEL_ALIASES[model]\n except KeyError as exc:\n known = \", \".join(sorted(PI_MODEL_ALIASES))\n raise ValueError(\n f\"Pi model '{model}' is unsupported; configured aliases: {known}\"\n ) from exc\n\n @staticmethod\n def _session_token(usage: Mapping[str, Any], key: str) -> int:\n value = usage.get(key)\n if type(value) is not int or value < 0:\n raise ValueError(\"assistant message has invalid token counts\")\n return value\n\n @contextmanager\n def execution_context(self, job: AgentJob) -> Iterator[None]:\n \"\"\"Install a private model catalog and disposable native session ledger.\"\"\"\n private_root = self._private_root(job)\n config_dir = private_root / \"config\"\n models_path = config_dir / \"models.json\"\n settings_path = config_dir / \"settings.json\"\n session_dir = self._session_dir(job)\n reserved = (private_root, config_dir, models_path, settings_path, session_dir)\n if any(path.is_symlink() for path in reserved):\n raise ValueError(\"Pi reserved configuration paths must not be symlinks\")\n if os.path.lexists(private_root):\n raise ValueError(\"run private directory contains a reserved Pi adapter path\")\n if any(not path.is_dir() for path in job.evidence_dirs):\n raise ValueError(\"Pi evidence paths must be directories\")\n if not os.environ.get(\"OPENROUTER_API_KEY\") and _read_openrouter_key_file() is None:\n raise ValueError(\n \"Pi lane needs an OpenRouter key: export OPENROUTER_API_KEY in the \"\n f\"launching shell or write the key to {OPENROUTER_KEY_FILE}\"\n )\n\n native_model = self._native_model(job.model.model)\n config_dir.mkdir(parents=True)\n self._active_config_dir = config_dir\n try:\n models_path.write_text(\n json.dumps(\n {\n \"providers\": {\n \"openrouter\": {\n \"baseUrl\": \"https://openrouter.ai/api/v1\",\n \"api\": \"openai-completions\",\n \"apiKey\": \"$OPENROUTER_API_KEY\",\n \"models\": [\n {\n \"id\": native_model,\n \"name\": \"GLM 5.2\",\n \"reasoning\": True,\n \"input\": [\"text\"],\n \"contextWindow\": 1_048_576,\n \"maxTokens\": 131_072,\n # OpenRouter's default (price-ordered) routing\n # re-rolls the provider on every turn, and some\n # hosts (Z.AI, Venice, partially Novita) buffer\n # whole responses instead of streaming. A long\n # write turn on a buffering host emits no bytes\n # for its entire generation, which reads as a\n # hang and previously tripped Pi's idle timeout.\n # Pin to hosts verified to stream tool-call\n # deltas, serve fp8, and allow >=128k completion\n # tokens.\n \"compat\": {\n \"openRouterRouting\": {\n \"order\": [\n \"baidu\",\n \"atlas-cloud\",\n \"akashml\",\n \"siliconflow\",\n \"streamlake\",\n ],\n \"allow_fallbacks\": False,\n }\n },\n }\n ],\n }\n }\n },\n sort_keys=True,\n )\n + \"\\n\",\n encoding=\"utf-8\",\n )\n # The pinned providers above stream continuously (observed\n # inter-chunk gaps of a few seconds), so a genuine multi-minute\n # silence means the stream is dead. Fail the turn at ten minutes\n # instead of hanging until the benchmark's outer timeout.\n settings_path.write_text(\n json.dumps({\"httpIdleTimeoutMs\": 600_000}, sort_keys=True) + \"\\n\",\n encoding=\"utf-8\",\n )\n yield\n finally:\n self._active_config_dir = None\n if private_root.is_symlink():\n private_root.unlink()\n elif private_root.is_dir():\n shutil.rmtree(private_root)\n elif os.path.lexists(private_root):\n raise ValueError(\"Pi reserved private root changed type\")\n\n def prepare_env(self, base: Mapping[str, str] | None = None) -> dict[str, str]:\n env = super().prepare_env(base)\n # Version probes run outside a job context and need no private catalog.\n if self._active_config_dir is not None:\n env[\"PI_CODING_AGENT_DIR\"] = str(self._active_config_dir)\n env[\"PI_TELEMETRY\"] = \"0\"\n if not env.get(\"OPENROUTER_API_KEY\"):\n key = _read_openrouter_key_file()\n if key is not None:\n env[\"OPENROUTER_API_KEY\"] = key\n return env\n\n def build_command(self, job: AgentJob) -> list[str]:\n native_model = self._native_model(job.model.model)\n if job.model.effort != \"high\":\n raise ValueError(\"Pi model 'glm-5.2' requires thinking effort 'high'\")\n return [\n self.resolve_binary(),\n \"--mode\",\n \"text\",\n \"--provider\",\n \"openrouter\",\n \"--model\",\n native_model,\n \"--thinking\",\n \"high\",\n \"--session-dir\",\n str(self._session_dir(job)),\n \"--no-extensions\",\n \"--no-skills\",\n \"--no-prompt-templates\",\n \"--approve\",\n \"--tools\",\n \"read,bash,edit,write,grep,find,ls\",\n \"-p\",\n f\"@{job.prompt_path}\",\n ]\n\n def parse_output(self, job: AgentJob, stdout_text: str) -> ParsedOutput:\n del stdout_text # Text mode is bounded; the native ledger is authoritative.\n session_dir = self._session_dir(job)\n if session_dir.is_symlink():\n raise ValueError(\"Pi session directory must not be a symlink\")\n if not session_dir.is_dir():\n return ParsedOutput()\n files = sorted(session_dir.glob(\"*.jsonl\"))\n if len(files) != 1:\n if not files:\n return ParsedOutput()\n raise ValueError(\"Pi usage capture must contain exactly one session\")\n session_path = files[0]\n if session_path.is_symlink() or not session_path.is_file():\n raise ValueError(\"Pi usage capture must be a regular file\")\n\n usage = Usage()\n assistant_messages = 0\n last_message: str | None = None\n session_id: str | None = None\n try:\n with session_path.open(encoding=\"utf-8\") as handle:\n for line in handle:\n if not line.strip():\n continue\n entry = json.loads(line)\n if not isinstance(entry, dict):\n raise ValueError(\"session entry must be an object\")\n if entry.get(\"type\") == \"session\":\n native_id = entry.get(\"id\")\n if (\n not isinstance(native_id, str)\n or not native_id\n or session_id is not None\n ):\n raise ValueError(\"session header is invalid\")\n session_id = native_id\n continue\n if entry.get(\"type\") != \"message\":\n continue\n message = entry.get(\"message\")\n if not isinstance(message, dict) or message.get(\"role\") != \"assistant\":\n continue\n native_usage = message.get(\"usage\")\n if not isinstance(native_usage, dict):\n raise ValueError(\"assistant message omitted usage\")\n usage = usage.add(\n Usage(\n input_tokens=self._session_token(native_usage, \"input\"),\n cached_input_tokens=self._session_token(native_usage, \"cacheRead\"),\n cache_write_tokens=self._session_token(native_usage, \"cacheWrite\"),\n output_tokens=self._session_token(native_usage, \"output\"),\n )\n )\n assistant_messages += 1\n content = message.get(\"content\")\n if isinstance(content, list):\n text_parts = [\n item[\"text\"]\n for item in content\n if isinstance(item, dict)\n and item.get(\"type\") == \"text\"\n and isinstance(item.get(\"text\"), str)\n ]\n if text_parts:\n last_message = \"\\n\".join(text_parts)\n except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:\n raise ValueError(\"Pi usage capture is unreadable\") from exc\n if session_id is None or assistant_messages == 0:\n return ParsedOutput()\n return ParsedOutput(\n usage=usage,\n # Custom model catalogs cannot safely encode OpenRouter's dynamic\n # routing price. The runner resolves cost from its pinned pricing.\n reported_cost_usd=None,\n last_message=last_message,\n session_id=session_id,\n )\n\n\nAGY_MODEL_ALIASES: Mapping[str, Mapping[str, str]] = {\n \"gemini-3.1-pro\": {\n \"low\": \"Gemini 3.1 Pro (Low)\",\n \"high\": \"Gemini 3.1 Pro (High)\",\n },\n \"gemini-3.5-flash\": {\n \"low\": \"Gemini 3.5 Flash (Low)\",\n \"medium\": \"Gemini 3.5 Flash (Medium)\",\n \"high\": \"Gemini 3.5 Flash (High)\",\n },\n}\n\n\n@register_harness\nclass AgyHarness(Harness):\n \"\"\"Google Antigravity CLI with its native terminal sandbox enabled.\"\"\"\n\n name = \"agy\"\n _COMPLETION_DIRECTIVE = (\n \"\\n\\nWork only by reading and editing files inside the assigned workspace. \"\n \"Do not call artifact creation, artifact export, or artifact presentation tools. \"\n \"Treat the remaining context as a hard budget: inventory paths once, use targeted searches \"\n \"and bounded reads, never dump large files or command output, and do not reread unchanged \"\n \"files. Implement the required deliverable directly and reserve enough output capacity for \"\n \"the final response. Once the required implementation is complete, run only brief local \"\n \"checks and stop. \"\n \"When the implementation and local checks are complete, return a concise plain-text summary \"\n \"and exit successfully.\\n\"\n )\n _PRIVATE_SUFFIX = \".agy-state\"\n _PROMPT_FILE = \"prompt.md\"\n _LAUNCHER_FILE = \"launch.py\"\n\n @classmethod\n def _private_root(cls, job: AgentJob) -> Path:\n return job.log_path.parent / f\".{job.log_path.name}{cls._PRIVATE_SUFFIX}\"\n\n @staticmethod\n def _native_model(model: str, effort: str) -> str:\n efforts = AGY_MODEL_ALIASES.get(model)\n if efforts is None:\n known = \", \".join(sorted(AGY_MODEL_ALIASES))\n raise ValueError(f\"AGY model '{model}' is unsupported; configured aliases: {known}\")\n try:\n return efforts[effort]\n except KeyError as exc:\n known = \", \".join(sorted(efforts))\n raise ValueError(\n f\"AGY model '{model}' does not support effort '{effort}'; choose: {known}\"\n ) from exc\n\n @contextmanager\n def execution_context(self, job: AgentJob) -> Iterator[None]:\n \"\"\"Stage private control files and disposable evidence outside the submission tree.\"\"\"\n state_dir = self._private_root(job)\n if state_dir.is_symlink() or state_dir.exists():\n raise ValueError(\"workspace contains a reserved AGY adapter path\")\n state_dir.mkdir(parents=True)\n try:\n (state_dir / self._LAUNCHER_FILE).write_text(\n \"import os\\n\"\n \"import sys\\n\"\n \"with open(sys.argv[1], encoding='utf-8') as fh:\\n\"\n \" prompt = fh.read()\\n\"\n \"os.execv(sys.argv[2], [sys.argv[2], *sys.argv[3:], '-p', prompt])\\n\",\n encoding=\"utf-8\",\n )\n prompt_text = job.prompt_path.read_text(encoding=\"utf-8\")\n for index, source in enumerate(job.evidence_dirs):\n if source.is_symlink() or not source.is_dir():\n raise ValueError(\"AGY evidence paths must be non-symlink directories\")\n if any(path.is_symlink() for path in source.rglob(\"*\")):\n raise ValueError(\"AGY evidence trees must not contain symlinks\")\n staged = state_dir / f\"evidence-{index}\"\n shutil.copytree(source, staged)\n prompt_text = prompt_text.replace(str(source), str(staged))\n (state_dir / self._PROMPT_FILE).write_text(\n prompt_text + self._COMPLETION_DIRECTIVE,\n encoding=\"utf-8\",\n )\n yield\n finally:\n if state_dir.is_symlink():\n state_dir.unlink()\n elif state_dir.exists():\n shutil.rmtree(state_dir)\n\n def build_command(self, job: AgentJob) -> list[str]:\n native_model = self._native_model(job.model.model, job.model.effort)\n state_dir = self._private_root(job)\n native_log_path = job.log_path.with_name(f\"{job.log_path.name}.agy\")\n cmd = [\n sys.executable,\n str(state_dir / self._LAUNCHER_FILE),\n str(state_dir / self._PROMPT_FILE),\n self.resolve_binary(),\n \"--model\",\n native_model,\n \"--mode\",\n \"accept-edits\",\n \"--output-format\",\n \"json\",\n \"--print-timeout\",\n \"24h\",\n \"--log-file\",\n str(native_log_path),\n \"--dangerously-skip-permissions\",\n \"--new-project\",\n \"--add-dir\",\n str(job.workdir),\n ]\n for index in range(len(job.evidence_dirs)):\n cmd.extend([\"--add-dir\", str(state_dir / f\"evidence-{index}\")])\n if job.sandbox_mode != \"danger-full-access\":\n cmd.append(\"--sandbox\")\n return cmd\n\n def parse_output(self, job: AgentJob, stdout_text: str) -> ParsedOutput:\n for obj in _iter_json_objects(stdout_text):\n status = obj.get(\"status\")\n if isinstance(status, str) and status != \"SUCCESS\":\n error = obj.get(\"error\")\n detail = error if isinstance(error, str) and error else status\n raise ValueError(f\"AGY execution failed: {detail}\")\n native_usage = obj.get(\"usage\")\n usage = None\n if isinstance(native_usage, dict):\n usage = Usage(\n input_tokens=_int_of(native_usage.get(\"input_tokens\")),\n output_tokens=_int_of(native_usage.get(\"output_tokens\")),\n )\n response = obj.get(\"response\")\n conversation_id = obj.get(\"conversation_id\")\n return ParsedOutput(\n usage=usage,\n reported_cost_usd=None,\n last_message=response if isinstance(response, str) else None,\n session_id=conversation_id if isinstance(conversation_id, str) else None,\n )\n return ParsedOutput()\n"}}}} |