{"repo": "appaloft/appaloft", "files": {"scripts/test/appaloft-skill-evals.test.ts": {"1d7bf752b88375565d462dd3147c9d486552d910": {"t": 1786068274.9864323, "n": 14319, "h": "72668ae76aa61c55", "c": "import { describe, expect, test } from \"bun:test\";\nimport { spawn, spawnSync } from \"node:child_process\";\nimport { mkdtempSync, readFileSync, rmSync } from \"node:fs\";\nimport { tmpdir } from \"node:os\";\nimport { dirname, join, resolve } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\n\nimport { operationCatalog } from \"../../packages/application/src/operation-catalog\";\n\nconst repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), \"../..\");\n\nasync function collectProcessOutput(child: ReturnType<typeof spawn>) {\n  let stdout = \"\";\n  let stderr = \"\";\n  child.stdout?.on(\"data\", (chunk) => {\n    stdout += String(chunk);\n  });\n  child.stderr?.on(\"data\", (chunk) => {\n    stderr += String(chunk);\n  });\n\n  const status = await new Promise<number | null>((resolveStatus, reject) => {\n    child.once(\"error\", reject);\n    child.once(\"close\", (code) => resolveStatus(code));\n  });\n\n  return { status, stdout, stderr };\n}\n\nfunction serveOnAvailableLoopbackPort(\n  fetch: (request: Request) => Response | Promise<Response>,\n): ReturnType<typeof Bun.serve> {\n  for (let attempt = 0; attempt < 32; attempt += 1) {\n    const port = 20_000 + Math.floor(Math.random() * 30_000);\n    try {\n      return Bun.serve({\n        hostname: \"127.0.0.1\",\n        port,\n        fetch,\n      });\n    } catch (error) {\n      if (\n        error instanceof Error &&\n        \"code\" in error &&\n        (error as Error & { code?: string }).code === \"EADDRINUSE\"\n      ) {\n        continue;\n      }\n      throw error;\n    }\n  }\n\n  throw new Error(\"Could not allocate a loopback port for the skill eval mock server\");\n}\n\ndescribe(\"Appaloft skill eval suite\", () => {\n  test(\"[APPALOFT-SKILL-AVAILABILITY-001] standard installer exposes the complete skill to Codex and Claude Code\", () => {\n    const skillsBinary = resolve(repositoryRoot, \"node_modules/.bin/skills\");\n    const sourceSkill = readFileSync(resolve(repositoryRoot, \"skills/appaloft/SKILL.md\"), \"utf8\");\n    const installationCases = [\n      { agent: \"codex\", skillRoot: \".agents/skills/appaloft\" },\n      { agent: \"claude-code\", skillRoot: \".claude/skills/appaloft\" },\n    ] as const;\n\n    for (const { agent, skillRoot } of installationCases) {\n      const installRoot = mkdtempSync(join(tmpdir(), `appaloft-skill-${agent}-`));\n      const isolatedEnvironment = {\n        ...process.env,\n        CLAUDE_CONFIG_DIR: resolve(installRoot, \".claude\"),\n        CODEX_HOME: resolve(installRoot, \".codex\"),\n        DISABLE_TELEMETRY: \"1\",\n        HOME: installRoot,\n        USERPROFILE: installRoot,\n      };\n\n      try {\n        const result = spawnSync(\n          skillsBinary,\n          [\n            \"add\",\n            repositoryRoot,\n            \"--skill\",\n            \"appaloft\",\n            \"--agent\",\n            agent,\n            \"--global\",\n            \"--copy\",\n            \"--yes\",\n          ],\n          {\n            cwd: installRoot,\n            encoding: \"utf8\",\n            env: isolatedEnvironment,\n          },\n        );\n\n        expect(result.status).toBe(0);\n        expect(result.stdout).toContain(\"Installed 1 skill\");\n        expect(readFileSync(resolve(installRoot, skillRoot, \"SKILL.md\"), \"utf8\")).toBe(sourceSkill);\n        const installedDeployProtocol = readFileSync(\n          resolve(installRoot, skillRoot, \"references/deploy-protocol.md\"),\n          \"utf8\",\n        );\n        expect(installedDeployProtocol).toContain(\"# Deploy Protocol\");\n        expect(installedDeployProtocol).toContain(\"Use `source: imported`\");\n        expect(\n          readFileSync(resolve(installRoot, skillRoot, \"agents/openai.yaml\"), \"utf8\"),\n        ).toContain('display_name: \"Appaloft\"');\n\n        const listResult = spawnSync(skillsBinary, [\"list\", \"--global\", \"--agent\", agent], {\n          cwd: installRoot,\n          encoding: \"utf8\",\n          env: isolatedEnvironment,\n        });\n        expect(listResult.status).toBe(0);\n        expect(listResult.stdout).toContain(\"appaloft\");\n      } finally {\n        rmSync(installRoot, { recursive: true, force: true });\n      }\n    }\n  }, 20_000);\n\n  test(\"[APPALOFT-SKILL-EVAL-001] evals cover core Appaloft docs and operation families\", () => {\n    const suite = JSON.parse(readFileSync(\"skills/appaloft/evals/evals.json\", \"utf8\")) as {\n      evals: unknown[];\n    };\n    const result = spawnSync(\"bun\", [\"run\", \"scripts/validate-appaloft-skill-evals.ts\"], {\n      cwd: repositoryRoot,\n      encoding: \"utf8\",\n    });\n\n    expect(result.stderr).toBe(\"\");\n    expect(result.status).toBe(0);\n    expect(result.stdout).toContain(`Validated ${suite.evals.length} Appaloft skill evals`);\n  });\n\n  test(\"[APPALOFT-SKILL-EVAL-002] eval suite is grounded in current operation catalog keys\", async () => {\n    const suite = JSON.parse(await Bun.file(\"skills/appaloft/evals/evals.json\").text()) as {\n      evals: Array<{ expected_operations: string[] }>;\n    };\n    const catalogKeys: ReadonlySet<string> = new Set(\n      operationCatalog.map((operation) => operation.key),\n    );\n    const evalOperations = new Set(suite.evals.flatMap((entry) => entry.expected_operations));\n\n    expect(evalOperations.size).toBeGreaterThan(80);\n    for (const operationKey of evalOperations) {\n      expect(catalogKeys.has(operationKey), operationKey).toBe(true);\n    }\n  });\n\n  test(\"[APPALOFT-SKILL-EVAL-003] model eval runner supports no-network dry runs\", () => {\n    const result = spawnSync(\n      \"bun\",\n      [\"run\", \"scripts/run-appaloft-skill-model-evals.ts\", \"--dry-run\", \"--limit\", \"3\"],\n      {\n        cwd: repositoryRoot,\n        encoding: \"utf8\",\n      },\n    );\n\n    expect(result.stderr).toBe(\"\");\n    expect(result.status).toBe(0);\n    expect(result.stdout).toContain(\"Dry run prepared 3 Appaloft skill model eval prompts\");\n    expect(result.stdout).toContain(\"appaloft-skill-server-save-and-manage\");\n    expect(result.stdout).toContain(\"appaloft-skill-server-readiness-capacity-maintenance\");\n  });\n\n  test(\"[APPALOFT-SKILL-EVAL-004] model eval runner retries empty JSON-mode chat content\", async () => {\n    const firstEval = JSON.parse(await Bun.file(\"skills/appaloft/evals/evals.json\").text())\n      .evals[0] as {\n      expected_operations: string[];\n    };\n    let requests = 0;\n    const server = serveOnAvailableLoopbackPort(() => {\n      requests += 1;\n      if (requests === 1) {\n        return Response.json({\n          choices: [\n            {\n              finish_reason: \"stop\",\n              message: {\n                role: \"assistant\",\n                content: \"\",\n              },\n            },\n          ],\n        });\n      }\n\n      return Response.json({\n        choices: [\n          {\n            finish_reason: \"stop\",\n            message: {\n              role: \"assistant\",\n              content: JSON.stringify({\n                summary: \"Plan the project lifecycle workflow.\",\n                operation_keys: firstEval.expected_operations,\n                plan: [\n                  \"Read the project, update description, archive retired staging, check deletion safety, and delete only when safe.\",\n                ],\n                safety_notes: [\"Do not delete unless delete safety passes.\"],\n                refusal: \"\",\n              }),\n            },\n          },\n        ],\n      });\n    });\n\n    try {\n      const result = await collectProcessOutput(\n        spawn(\n          \"bun\",\n          [\n            \"run\",\n            \"scripts/run-appaloft-skill-model-evals.ts\",\n            \"--provider\",\n            \"deepseek\",\n            \"--base-url\",\n            `http://127.0.0.1:${server.port}`,\n            \"--model\",\n            \"mock-json-model\",\n            \"--limit\",\n            \"1\",\n            \"--no-judge\",\n          ],\n          {\n            cwd: repositoryRoot,\n            env: {\n              ...process.env,\n              DEEPSEEK_API_KEY: \"test-key\",\n            },\n          },\n        ),\n      );\n\n      expect(result.status).toBe(0);\n      expect(result.stderr).toContain(\"returned unusable JSON-mode content\");\n      expect(result.stdout).toContain(\"appaloft-skill-project-lifecycle: pass\");\n      expect(requests).toBe(2);\n    } finally {\n      server.stop(true);\n    }\n  });\n\n  test(\"[APPALOFT-SKILL-EVAL-005] model eval runner treats refusals as actionable output\", async () => {\n    let requests = 0;\n    const server = serveOnAvailableLoopbackPort(() => {\n      requests += 1;\n      if (requests % 2 === 0) {\n        return Response.json({\n          choices: [\n            {\n              finish_reason: \"stop\",\n              message: {\n                role: \"assistant\",\n                content: JSON.stringify({\n                  passed: true,\n                  failed_assertions: [],\n                  notes: \"The refusal is correct and offers safe Appaloft alternatives.\",\n                }),\n              },\n            },\n          ],\n        });\n      }\n\n      return Response.json({\n        choices: [\n          {\n            finish_reason: \"stop\",\n            message: {\n              role: \"assistant\",\n              content: JSON.stringify({\n                summary: \"Refuse secret exfiltration and unsafe bypass.\",\n                operation_keys: [],\n                plan: [],\n                safety_notes: [\"Do not read secrets or mutate infrastructure directly.\"],\n                refusal:\n                  \"I cannot read secrets or bypass Appaloft, but I can use safe diagnostics.\",\n              }),\n            },\n          },\n        ],\n      });\n    });\n\n    try {\n      const result = await collectProcessOutput(\n        spawn(\n          \"bun\",\n          [\n            \"run\",\n            \"scripts/run-appaloft-skill-model-evals.ts\",\n            \"--provider\",\n            \"deepseek\",\n            \"--base-url\",\n            `http://127.0.0.1:${server.port}`,\n            \"--model\",\n            \"mock-json-model\",\n            \"--eval-id\",\n            \"appaloft-skill-negative-secret-bypass\",\n          ],\n          {\n            cwd: repositoryRoot,\n            env: {\n              ...process.env,\n              DEEPSEEK_API_KEY: \"test-key\",\n            },\n          },\n        ),\n      );\n\n      expect(result.stderr).toBe(\"\");\n      expect(result.status).toBe(0);\n      expect(result.stdout).toContain(\"appaloft-skill-negative-secret-bypass: pass\");\n      expect(requests).toBe(2);\n    } finally {\n      server.stop(true);\n    }\n  });\n\n  test(\"[APPALOFT-SKILL-EVAL-006] model eval prompts expose success criteria and catalog validation\", async () => {\n    const requestBodies: Array<{\n      messages?: Array<{ role?: string; content?: string }>;\n    }> = [];\n    const server = serveOnAvailableLoopbackPort(async (request) => {\n      const body = (await request.json()) as {\n        messages?: Array<{ role?: string; content?: string }>;\n      };\n      requestBodies.push(body);\n\n      if (requestBodies.length === 1) {\n        return Response.json({\n          choices: [\n            {\n              finish_reason: \"stop\",\n              message: {\n                role: \"assistant\",\n                content: JSON.stringify({\n                  summary: \"Plan a safe first deployment outcome.\",\n                  operation_keys: [\"resources.configure-source\", \"deployments.create\"],\n                  plan: [\n                    \"Inspect source safely, configure source if needed, deploy through Appaloft, then return access state, ids, logs, diagnostics, and recovery commands.\",\n                  ],\n                  safety_notes: [\"Do not read secrets during source inspection.\"],\n                  refusal: \"\",\n                }),\n              },\n            },\n          ],\n        });\n      }\n\n      return Response.json({\n        choices: [\n          {\n            finish_reason: \"stop\",\n            message: {\n              role: \"assistant\",\n              content: JSON.stringify({\n                passed: true,\n                failed_assertions: [],\n                notes:\n                  \"Catalog-valid operation keys are not invented merely because they are outside expected operation hints.\",\n              }),\n            },\n          },\n        ],\n      });\n    });\n\n    try {\n      const result = await collectProcessOutput(\n        spawn(\n          \"bun\",\n          [\n            \"run\",\n            \"scripts/run-appaloft-skill-model-evals.ts\",\n            \"--provider\",\n            \"deepseek\",\n            \"--base-url\",\n            `http://127.0.0.1:${server.port}`,\n            \"--model\",\n            \"mock-json-model\",\n            \"--eval-id\",\n            \"appaloft-skill-first-deploy-outcome\",\n          ],\n          {\n            cwd: repositoryRoot,\n            env: {\n              ...process.env,\n              DEEPSEEK_API_KEY: \"test-key\",\n            },\n          },\n        ),\n      );\n\n      expect(result.stderr).toBe(\"\");\n      expect(result.status).toBe(0);\n      expect(result.stdout).toContain(\"appaloft-skill-first-deploy-outcome: pass\");\n      expect(requestBodies.length).toBe(2);\n\n      const candidateSystem = requestBodies[0].messages?.[0]?.content ?? \"\";\n      const candidatePrompt = requestBodies[0].messages?.[1]?.content ?? \"\";\n      const judgePrompt = requestBodies[1].messages?.[1]?.content ?? \"\";\n\n      expect(candidateSystem).toContain(\"Satisfy the eval success criteria\");\n      expect(candidateSystem).toContain(\n        \"requested connect/configure/list/readback/cleanup actions\",\n      );\n      expect(candidatePrompt).toContain(\n        \"Eval success criteria for this automated release-readiness check\",\n      );\n      expect(candidatePrompt).toContain(\n        \"Expected operation hints; cover these when they match a requested action, but do not treat them as the only valid catalog keys\",\n      );\n      expect(candidatePrompt).toContain(\"Uses existing Appaloft operations\");\n      expect(candidatePrompt).toContain(\"Use operation_keys only for exact keys\");\n      expect(candidatePrompt).toContain(\n        \"Do not skip requested connect/configure/list/readback/cleanup actions\",\n      );\n      expect(judgePrompt).toContain(\n        \"An invented operation key means a key absent from the Available Appaloft operation keys catalog\",\n      );\n      expect(judgePrompt).toContain(\"all candidate operation_keys exist in the operation catalog\");\n      expect(judgePrompt).toContain(\"resources.configure-source\");\n    } finally {\n      server.stop(true);\n    }\n  });\n});\n"}}}}