1 line
114 KiB
JSON
1 line
114 KiB
JSON
{"repo": "asle149/baton-demo", "files": {"server/src/test/java/com/baton/bench/ClaimExtractionBenchTest.java": {"61fed40a8b5d4e25249c7c372d6084d9a85a2939": {"t": 1785745705.3351266, "n": 12997, "h": "ef322d1658fe08cb", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmBudgetGuard;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiCompatClient;\nimport com.baton.adapter.out.llm.OpenAiCompatibleLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ingest.ExtractClaimsPort;\nimport com.baton.application.port.out.shared.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.Counter;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·비용·시간 비교.\n * 실행: ./gradlew benchTest. 실호출 후보는 키가 설정된 항목만 순차 실행한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\tprivate static final double MIN_F1 = 0.70;\n\t// kimi-k3는 단가($3/$15) 사유로 기본 후보에서 제외 — 필요 시 BENCH_COMPAT_MODELS로만 추가\n\tprivate static final String DEFAULT_COMPAT_MODELS = String.join(\";\",\n\t\t\t\"deepseek|https://api.deepseek.com|DEEPSEEK_API_KEY|deepseek-v4-pro|disabled\",\n\t\t\t\"upstage|https://api.upstage.ai/v1|UPSTAGE_API_KEY|solar-pro3\");\n\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble precision = precision();\n\t\t\tdouble recall = recall();\n\t\t\treturn precision + recall == 0 ? 0.0 : 2 * precision * recall / (precision + recall);\n\t\t}\n\t}\n\n\trecord ModelCost(double inputPerMtok, double outputPerMtok) {\n\n\t\tdouble usd(long inputTokens, long outputTokens) {\n\t\t\treturn (inputTokens * inputPerMtok + outputTokens * outputPerMtok) / 1_000_000.0;\n\t\t}\n\t}\n\n\trecord CompatCandidate(String label, String baseUrl, String keyEnv, String model, String thinking,\n\t\t\tModelCost cost) {\n\t}\n\n\trecord RunResult(Map<String, Tally> byCategory, Tally overall, long calls,\n\t\t\tlong inputTokens, long outputTokens, long reasoningTokens, double costUsd, long elapsedMillis) {\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tdouble maxUsd = envDouble(\"BENCH_MAX_USD\", 2.0);\n\t\tBenchBudget budget = new BenchBudget(maxUsd);\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장, 지출 상한 $\"\n\t\t\t\t+ \"%.2f\".formatted(maxUsd) + \")\");\n\t\tprintHeader();\n\n\t\tRunResult ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()),\n\t\t\t\tcases, null, new ModelCost(0, 0));\n\t\tprintRow(\"-\", \"rule\", ruleResult);\n\n\t\tfor (CompatCandidate candidate : compatCandidates()) {\n\t\t\tif (budget.exhausted()) {\n\t\t\t\tprintBudgetStop(budget);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tString apiKey = System.getenv(candidate.keyEnv());\n\t\t\tif (apiKey == null || apiKey.isBlank()) {\n\t\t\t\tSystem.out.println(\"건너뜀: \" + candidate.label() + \" (\" + candidate.keyEnv() + \" 없음)\");\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\trunLlmStrategies(candidate.label(),\n\t\t\t\t\tmeters -> compatAdapter(candidate, apiKey, meters), candidate.cost(), cases, budget);\n\t\t}\n\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tif (bedrockModels != null && !bedrockModels.isBlank() && !budget.exhausted()) {\n\t\t\ttry (BedrockRuntimeClient bedrock = BedrockLlmAdapter.buildClient(\n\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"us-east-1\"))) {\n\t\t\t\tfor (String rawModelId : bedrockModels.split(\",\")) {\n\t\t\t\t\tif (budget.exhausted()) {\n\t\t\t\t\t\tprintBudgetStop(budget);\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\tString modelId = rawModelId.strip();\n\t\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId,\n\t\t\t\t\t\t\tmeters -> new BedrockLlmAdapter(bedrock, modelId, meters),\n\t\t\t\t\t\t\tnew ModelCost(1.0, 5.0), cases, budget);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tSystem.out.printf(\"누적 추정 비용: $%.4f / $%.2f%n\", budget.spentUsd(), budget.maxUsd());\n\t\tassertEquals(1.0, ruleResult.byCategory().get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.byCategory().get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\tprivate void runLlmStrategies(String modelLabel, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases, BenchBudget budget) {\n\t\tRunResult llm = runWithMeters(\"llm\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"llm\", llm);\n\t\tbudget.add(llm.costUsd());\n\t\tif (budget.exhausted()) {\n\t\t\treturn;\n\t\t}\n\n\t\tRunResult hybrid = runWithMeters(\"rule-first\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"rule-first\", hybrid);\n\t\tbudget.add(hybrid.costUsd());\n\t}\n\n\tprivate RunResult runWithMeters(String strategy, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases) {\n\t\tSimpleMeterRegistry meters = new SimpleMeterRegistry();\n\t\tLlmClaimExtractor llmExtractor = new LlmClaimExtractor(\n\t\t\t\tllmFactory.apply(meters), objectMapper, meters);\n\t\tExtractClaimsPort extractor = \"rule-first\".equals(strategy)\n\t\t\t\t? new RuleFirstClaimExtractor(new RuleClaimExtractor(), llmExtractor)\n\t\t\t\t: llmExtractor;\n\t\treturn run(strategy, extractor, cases, meters, cost);\n\t}\n\n\tprivate RunResult run(String strategy, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters, ModelCost cost) {\n\t\tlong startedAt = System.nanoTime();\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tTally row = new Tally((int) tp, fp, fn);\n\t\t\tbyCategory.merge(benchCase.category(), row,\n\t\t\t\t\t(left, right) -> left.plus(right.tp(), right.fp(), right.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tlong elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000;\n\t\tlong calls = meters == null ? 0 : Math.round(\n\t\t\t\tmeters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count());\n\t\tlong inputTokens = sumCounters(meters, \"baton.llm.tokens.input\");\n\t\tlong outputTokens = sumCounters(meters, \"baton.llm.tokens.output\");\n\t\tlong reasoningTokens = sumCounters(meters, \"baton.llm.tokens.reasoning\");\n\t\treturn new RunResult(Map.copyOf(byCategory), overall, calls, inputTokens, outputTokens,\n\t\t\t\treasoningTokens, cost.usd(inputTokens, outputTokens), elapsedMillis);\n\t}\n\n\tprivate LlmPort compatAdapter(CompatCandidate candidate, String apiKey, SimpleMeterRegistry meters) {\n\t\tString endpointPath = candidate.baseUrl().replaceAll(\"/+$\", \"\").endsWith(\"/v1\")\n\t\t\t\t? \"/chat/completions\" : \"/v1/chat/completions\";\n\t\tMap<String, Object> extraBody = candidate.thinking().isBlank()\n\t\t\t\t? Map.of() : Map.of(\"thinking\", Map.of(\"type\", candidate.thinking()));\n\t\tOpenAiCompatClient client = new OpenAiCompatClient(apiKey, candidate.model(),\n\t\t\t\tcandidate.baseUrl(), endpointPath, extraBody);\n\t\treturn new BenchCompatAdapter(candidate.label(), client, meters);\n\t}\n\n\tprivate List<CompatCandidate> compatCandidates() {\n\t\tString configured = System.getenv().getOrDefault(\"BENCH_COMPAT_MODELS\", DEFAULT_COMPAT_MODELS);\n\t\tList<CompatCandidate> candidates = new ArrayList<>();\n\t\tfor (String rawCandidate : configured.split(\";\")) {\n\t\t\tif (rawCandidate.isBlank()) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tString[] fields = rawCandidate.split(\"\\\\|\", -1);\n\t\t\tif (fields.length < 4 || fields.length > 5) {\n\t\t\t\tthrow new IllegalArgumentException(\n\t\t\t\t\t\t\"BENCH_COMPAT_MODELS 형식 오류: 라벨|base-url|키env|모델[|thinking]\");\n\t\t\t}\n\t\t\tString model = fields[3].strip();\n\t\t\tcandidates.add(new CompatCandidate(fields[0].strip(), fields[1].strip(), fields[2].strip(),\n\t\t\t\t\tmodel, fields.length == 5 ? fields[4].strip() : \"\", modelCost(model)));\n\t\t}\n\t\treturn List.copyOf(candidates);\n\t}\n\n\tprivate ModelCost modelCost(String model) {\n\t\tString normalized = model.toLowerCase();\n\t\tif (normalized.contains(\"kimi-k3\")) {\n\t\t\treturn new ModelCost(3.0, 15.0);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-flash\")) {\n\t\t\treturn new ModelCost(0.14, 0.28);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-pro\")) {\n\t\t\treturn new ModelCost(0.435, 0.87);\n\t\t}\n\t\tif (normalized.contains(\"solar-pro3\") || normalized.contains(\"gpt-4o-mini\")) {\n\t\t\treturn new ModelCost(0.15, 0.60);\n\t\t}\n\t\tthrow new IllegalArgumentException(\"벤치 단가 매핑이 없는 모델입니다: \" + model);\n\t}\n\n\tprivate void printHeader() {\n\t\tSystem.out.printf(\"%-28s %-12s %7s %7s %7s %7s %10s %10s %10s %9s %9s%n\",\n\t\t\t\t\"모델\", \"전략\", \"정밀도\", \"재현율\", \"F1\", \"호출\", \"입력토큰\", \"출력토큰\", \"추론토큰\", \"비용USD\", \"시간ms\");\n\t}\n\n\tprivate void printRow(String model, String strategy, RunResult result) {\n\t\tString verdict = \"rule\".equals(strategy) ? \"기준외\"\n\t\t\t\t: result.overall().f1() >= MIN_F1 ? \"통과\" : \"미달\";\n\t\tSystem.out.printf(\"%-28s %-12s %7.2f %7.2f %7.2f %7d %10d %10d %10d %9.4f %9d %s%n\",\n\t\t\t\tmodel, strategy, result.overall().precision(), result.overall().recall(),\n\t\t\t\tresult.overall().f1(), result.calls(), result.inputTokens(), result.outputTokens(),\n\t\t\t\tresult.reasoningTokens(), result.costUsd(), result.elapsedMillis(), verdict);\n\t}\n\n\tprivate void printBudgetStop(BenchBudget budget) {\n\t\tSystem.out.printf(\"중단: BENCH_MAX_USD 도달 ($%.4f / $%.2f), 남은 후보를 실행하지 않습니다.%n\",\n\t\t\t\tbudget.spentUsd(), budget.maxUsd());\n\t}\n\n\tprivate long sumCounters(SimpleMeterRegistry meters, String name) {\n\t\tif (meters == null) {\n\t\t\treturn 0;\n\t\t}\n\t\treturn Math.round(meters.find(name).counters().stream()\n\t\t\t\t.filter(counter -> counter.getId().getTag(\"provider\") != null)\n\t\t\t\t.mapToDouble(Counter::count).sum());\n\t}\n\n\tprivate double envDouble(String name, double defaultValue) {\n\t\tString value = System.getenv(name);\n\t\tdouble parsed = value == null || value.isBlank() ? defaultValue : Double.parseDouble(value);\n\t\tif (parsed < 0) {\n\t\t\tthrow new IllegalArgumentException(name + \"은 0 이상이어야 합니다\");\n\t\t}\n\t\treturn parsed;\n\t}\n\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> normalize = value -> value == null ? \"\" : value.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", normalize.apply(claim.subject()), normalize.apply(claim.attribute()),\n\t\t\t\tnormalize.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode claim = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", claim.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"attribute\").asString(\"\"), claim.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n\n\tprivate static final class BenchCompatAdapter extends OpenAiCompatibleLlmAdapter {\n\n\t\tprivate BenchCompatAdapter(String provider, OpenAiCompatClient client, SimpleMeterRegistry meters) {\n\t\t\tsuper(provider, client, LlmBudgetGuard.unlimited(meters));\n\t\t}\n\t}\n\n\tprivate static final class BenchBudget {\n\n\t\tprivate final double maxUsd;\n\t\tprivate double spentUsd;\n\n\t\tprivate BenchBudget(double maxUsd) {\n\t\t\tthis.maxUsd = maxUsd;\n\t\t}\n\n\t\tprivate void add(double usd) {\n\t\t\tspentUsd += usd;\n\t\t}\n\n\t\tprivate boolean exhausted() {\n\t\t\treturn spentUsd >= maxUsd;\n\t\t}\n\n\t\tprivate double spentUsd() {\n\t\t\treturn spentUsd;\n\t\t}\n\n\t\tprivate double maxUsd() {\n\t\t\treturn maxUsd;\n\t\t}\n\t}\n}\n"}, "bc8197bd5e534abd8ae43ce94eeab4b35d5eb543": {"t": 1785755764.0851774, "n": 12997, "h": "ef322d1658fe08cb", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmBudgetGuard;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiCompatClient;\nimport com.baton.adapter.out.llm.OpenAiCompatibleLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ingest.ExtractClaimsPort;\nimport com.baton.application.port.out.shared.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.Counter;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·비용·시간 비교.\n * 실행: ./gradlew benchTest. 실호출 후보는 키가 설정된 항목만 순차 실행한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\tprivate static final double MIN_F1 = 0.70;\n\t// kimi-k3는 단가($3/$15) 사유로 기본 후보에서 제외 — 필요 시 BENCH_COMPAT_MODELS로만 추가\n\tprivate static final String DEFAULT_COMPAT_MODELS = String.join(\";\",\n\t\t\t\"deepseek|https://api.deepseek.com|DEEPSEEK_API_KEY|deepseek-v4-pro|disabled\",\n\t\t\t\"upstage|https://api.upstage.ai/v1|UPSTAGE_API_KEY|solar-pro3\");\n\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble precision = precision();\n\t\t\tdouble recall = recall();\n\t\t\treturn precision + recall == 0 ? 0.0 : 2 * precision * recall / (precision + recall);\n\t\t}\n\t}\n\n\trecord ModelCost(double inputPerMtok, double outputPerMtok) {\n\n\t\tdouble usd(long inputTokens, long outputTokens) {\n\t\t\treturn (inputTokens * inputPerMtok + outputTokens * outputPerMtok) / 1_000_000.0;\n\t\t}\n\t}\n\n\trecord CompatCandidate(String label, String baseUrl, String keyEnv, String model, String thinking,\n\t\t\tModelCost cost) {\n\t}\n\n\trecord RunResult(Map<String, Tally> byCategory, Tally overall, long calls,\n\t\t\tlong inputTokens, long outputTokens, long reasoningTokens, double costUsd, long elapsedMillis) {\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tdouble maxUsd = envDouble(\"BENCH_MAX_USD\", 2.0);\n\t\tBenchBudget budget = new BenchBudget(maxUsd);\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장, 지출 상한 $\"\n\t\t\t\t+ \"%.2f\".formatted(maxUsd) + \")\");\n\t\tprintHeader();\n\n\t\tRunResult ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()),\n\t\t\t\tcases, null, new ModelCost(0, 0));\n\t\tprintRow(\"-\", \"rule\", ruleResult);\n\n\t\tfor (CompatCandidate candidate : compatCandidates()) {\n\t\t\tif (budget.exhausted()) {\n\t\t\t\tprintBudgetStop(budget);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tString apiKey = System.getenv(candidate.keyEnv());\n\t\t\tif (apiKey == null || apiKey.isBlank()) {\n\t\t\t\tSystem.out.println(\"건너뜀: \" + candidate.label() + \" (\" + candidate.keyEnv() + \" 없음)\");\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\trunLlmStrategies(candidate.label(),\n\t\t\t\t\tmeters -> compatAdapter(candidate, apiKey, meters), candidate.cost(), cases, budget);\n\t\t}\n\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tif (bedrockModels != null && !bedrockModels.isBlank() && !budget.exhausted()) {\n\t\t\ttry (BedrockRuntimeClient bedrock = BedrockLlmAdapter.buildClient(\n\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"us-east-1\"))) {\n\t\t\t\tfor (String rawModelId : bedrockModels.split(\",\")) {\n\t\t\t\t\tif (budget.exhausted()) {\n\t\t\t\t\t\tprintBudgetStop(budget);\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\tString modelId = rawModelId.strip();\n\t\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId,\n\t\t\t\t\t\t\tmeters -> new BedrockLlmAdapter(bedrock, modelId, meters),\n\t\t\t\t\t\t\tnew ModelCost(1.0, 5.0), cases, budget);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tSystem.out.printf(\"누적 추정 비용: $%.4f / $%.2f%n\", budget.spentUsd(), budget.maxUsd());\n\t\tassertEquals(1.0, ruleResult.byCategory().get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.byCategory().get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\tprivate void runLlmStrategies(String modelLabel, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases, BenchBudget budget) {\n\t\tRunResult llm = runWithMeters(\"llm\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"llm\", llm);\n\t\tbudget.add(llm.costUsd());\n\t\tif (budget.exhausted()) {\n\t\t\treturn;\n\t\t}\n\n\t\tRunResult hybrid = runWithMeters(\"rule-first\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"rule-first\", hybrid);\n\t\tbudget.add(hybrid.costUsd());\n\t}\n\n\tprivate RunResult runWithMeters(String strategy, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases) {\n\t\tSimpleMeterRegistry meters = new SimpleMeterRegistry();\n\t\tLlmClaimExtractor llmExtractor = new LlmClaimExtractor(\n\t\t\t\tllmFactory.apply(meters), objectMapper, meters);\n\t\tExtractClaimsPort extractor = \"rule-first\".equals(strategy)\n\t\t\t\t? new RuleFirstClaimExtractor(new RuleClaimExtractor(), llmExtractor)\n\t\t\t\t: llmExtractor;\n\t\treturn run(strategy, extractor, cases, meters, cost);\n\t}\n\n\tprivate RunResult run(String strategy, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters, ModelCost cost) {\n\t\tlong startedAt = System.nanoTime();\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tTally row = new Tally((int) tp, fp, fn);\n\t\t\tbyCategory.merge(benchCase.category(), row,\n\t\t\t\t\t(left, right) -> left.plus(right.tp(), right.fp(), right.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tlong elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000;\n\t\tlong calls = meters == null ? 0 : Math.round(\n\t\t\t\tmeters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count());\n\t\tlong inputTokens = sumCounters(meters, \"baton.llm.tokens.input\");\n\t\tlong outputTokens = sumCounters(meters, \"baton.llm.tokens.output\");\n\t\tlong reasoningTokens = sumCounters(meters, \"baton.llm.tokens.reasoning\");\n\t\treturn new RunResult(Map.copyOf(byCategory), overall, calls, inputTokens, outputTokens,\n\t\t\t\treasoningTokens, cost.usd(inputTokens, outputTokens), elapsedMillis);\n\t}\n\n\tprivate LlmPort compatAdapter(CompatCandidate candidate, String apiKey, SimpleMeterRegistry meters) {\n\t\tString endpointPath = candidate.baseUrl().replaceAll(\"/+$\", \"\").endsWith(\"/v1\")\n\t\t\t\t? \"/chat/completions\" : \"/v1/chat/completions\";\n\t\tMap<String, Object> extraBody = candidate.thinking().isBlank()\n\t\t\t\t? Map.of() : Map.of(\"thinking\", Map.of(\"type\", candidate.thinking()));\n\t\tOpenAiCompatClient client = new OpenAiCompatClient(apiKey, candidate.model(),\n\t\t\t\tcandidate.baseUrl(), endpointPath, extraBody);\n\t\treturn new BenchCompatAdapter(candidate.label(), client, meters);\n\t}\n\n\tprivate List<CompatCandidate> compatCandidates() {\n\t\tString configured = System.getenv().getOrDefault(\"BENCH_COMPAT_MODELS\", DEFAULT_COMPAT_MODELS);\n\t\tList<CompatCandidate> candidates = new ArrayList<>();\n\t\tfor (String rawCandidate : configured.split(\";\")) {\n\t\t\tif (rawCandidate.isBlank()) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tString[] fields = rawCandidate.split(\"\\\\|\", -1);\n\t\t\tif (fields.length < 4 || fields.length > 5) {\n\t\t\t\tthrow new IllegalArgumentException(\n\t\t\t\t\t\t\"BENCH_COMPAT_MODELS 형식 오류: 라벨|base-url|키env|모델[|thinking]\");\n\t\t\t}\n\t\t\tString model = fields[3].strip();\n\t\t\tcandidates.add(new CompatCandidate(fields[0].strip(), fields[1].strip(), fields[2].strip(),\n\t\t\t\t\tmodel, fields.length == 5 ? fields[4].strip() : \"\", modelCost(model)));\n\t\t}\n\t\treturn List.copyOf(candidates);\n\t}\n\n\tprivate ModelCost modelCost(String model) {\n\t\tString normalized = model.toLowerCase();\n\t\tif (normalized.contains(\"kimi-k3\")) {\n\t\t\treturn new ModelCost(3.0, 15.0);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-flash\")) {\n\t\t\treturn new ModelCost(0.14, 0.28);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-pro\")) {\n\t\t\treturn new ModelCost(0.435, 0.87);\n\t\t}\n\t\tif (normalized.contains(\"solar-pro3\") || normalized.contains(\"gpt-4o-mini\")) {\n\t\t\treturn new ModelCost(0.15, 0.60);\n\t\t}\n\t\tthrow new IllegalArgumentException(\"벤치 단가 매핑이 없는 모델입니다: \" + model);\n\t}\n\n\tprivate void printHeader() {\n\t\tSystem.out.printf(\"%-28s %-12s %7s %7s %7s %7s %10s %10s %10s %9s %9s%n\",\n\t\t\t\t\"모델\", \"전략\", \"정밀도\", \"재현율\", \"F1\", \"호출\", \"입력토큰\", \"출력토큰\", \"추론토큰\", \"비용USD\", \"시간ms\");\n\t}\n\n\tprivate void printRow(String model, String strategy, RunResult result) {\n\t\tString verdict = \"rule\".equals(strategy) ? \"기준외\"\n\t\t\t\t: result.overall().f1() >= MIN_F1 ? \"통과\" : \"미달\";\n\t\tSystem.out.printf(\"%-28s %-12s %7.2f %7.2f %7.2f %7d %10d %10d %10d %9.4f %9d %s%n\",\n\t\t\t\tmodel, strategy, result.overall().precision(), result.overall().recall(),\n\t\t\t\tresult.overall().f1(), result.calls(), result.inputTokens(), result.outputTokens(),\n\t\t\t\tresult.reasoningTokens(), result.costUsd(), result.elapsedMillis(), verdict);\n\t}\n\n\tprivate void printBudgetStop(BenchBudget budget) {\n\t\tSystem.out.printf(\"중단: BENCH_MAX_USD 도달 ($%.4f / $%.2f), 남은 후보를 실행하지 않습니다.%n\",\n\t\t\t\tbudget.spentUsd(), budget.maxUsd());\n\t}\n\n\tprivate long sumCounters(SimpleMeterRegistry meters, String name) {\n\t\tif (meters == null) {\n\t\t\treturn 0;\n\t\t}\n\t\treturn Math.round(meters.find(name).counters().stream()\n\t\t\t\t.filter(counter -> counter.getId().getTag(\"provider\") != null)\n\t\t\t\t.mapToDouble(Counter::count).sum());\n\t}\n\n\tprivate double envDouble(String name, double defaultValue) {\n\t\tString value = System.getenv(name);\n\t\tdouble parsed = value == null || value.isBlank() ? defaultValue : Double.parseDouble(value);\n\t\tif (parsed < 0) {\n\t\t\tthrow new IllegalArgumentException(name + \"은 0 이상이어야 합니다\");\n\t\t}\n\t\treturn parsed;\n\t}\n\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> normalize = value -> value == null ? \"\" : value.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", normalize.apply(claim.subject()), normalize.apply(claim.attribute()),\n\t\t\t\tnormalize.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode claim = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", claim.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"attribute\").asString(\"\"), claim.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n\n\tprivate static final class BenchCompatAdapter extends OpenAiCompatibleLlmAdapter {\n\n\t\tprivate BenchCompatAdapter(String provider, OpenAiCompatClient client, SimpleMeterRegistry meters) {\n\t\t\tsuper(provider, client, LlmBudgetGuard.unlimited(meters));\n\t\t}\n\t}\n\n\tprivate static final class BenchBudget {\n\n\t\tprivate final double maxUsd;\n\t\tprivate double spentUsd;\n\n\t\tprivate BenchBudget(double maxUsd) {\n\t\t\tthis.maxUsd = maxUsd;\n\t\t}\n\n\t\tprivate void add(double usd) {\n\t\t\tspentUsd += usd;\n\t\t}\n\n\t\tprivate boolean exhausted() {\n\t\t\treturn spentUsd >= maxUsd;\n\t\t}\n\n\t\tprivate double spentUsd() {\n\t\t\treturn spentUsd;\n\t\t}\n\n\t\tprivate double maxUsd() {\n\t\t\treturn maxUsd;\n\t\t}\n\t}\n}\n"}, "c78f987a3e5a71f42318a7fb20819544ee6c7930": {"t": 1785755765.5071483, "n": 12983, "h": "5f7aeece2a5acc38", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmBudgetGuard;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiCompatClient;\nimport com.baton.adapter.out.llm.OpenAiCompatibleLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.application.port.out.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.Counter;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·비용·시간 비교.\n * 실행: ./gradlew benchTest. 실호출 후보는 키가 설정된 항목만 순차 실행한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\tprivate static final double MIN_F1 = 0.70;\n\t// kimi-k3는 단가($3/$15) 사유로 기본 후보에서 제외 — 필요 시 BENCH_COMPAT_MODELS로만 추가\n\tprivate static final String DEFAULT_COMPAT_MODELS = String.join(\";\",\n\t\t\t\"deepseek|https://api.deepseek.com|DEEPSEEK_API_KEY|deepseek-v4-pro|disabled\",\n\t\t\t\"upstage|https://api.upstage.ai/v1|UPSTAGE_API_KEY|solar-pro3\");\n\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble precision = precision();\n\t\t\tdouble recall = recall();\n\t\t\treturn precision + recall == 0 ? 0.0 : 2 * precision * recall / (precision + recall);\n\t\t}\n\t}\n\n\trecord ModelCost(double inputPerMtok, double outputPerMtok) {\n\n\t\tdouble usd(long inputTokens, long outputTokens) {\n\t\t\treturn (inputTokens * inputPerMtok + outputTokens * outputPerMtok) / 1_000_000.0;\n\t\t}\n\t}\n\n\trecord CompatCandidate(String label, String baseUrl, String keyEnv, String model, String thinking,\n\t\t\tModelCost cost) {\n\t}\n\n\trecord RunResult(Map<String, Tally> byCategory, Tally overall, long calls,\n\t\t\tlong inputTokens, long outputTokens, long reasoningTokens, double costUsd, long elapsedMillis) {\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tdouble maxUsd = envDouble(\"BENCH_MAX_USD\", 2.0);\n\t\tBenchBudget budget = new BenchBudget(maxUsd);\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장, 지출 상한 $\"\n\t\t\t\t+ \"%.2f\".formatted(maxUsd) + \")\");\n\t\tprintHeader();\n\n\t\tRunResult ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()),\n\t\t\t\tcases, null, new ModelCost(0, 0));\n\t\tprintRow(\"-\", \"rule\", ruleResult);\n\n\t\tfor (CompatCandidate candidate : compatCandidates()) {\n\t\t\tif (budget.exhausted()) {\n\t\t\t\tprintBudgetStop(budget);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tString apiKey = System.getenv(candidate.keyEnv());\n\t\t\tif (apiKey == null || apiKey.isBlank()) {\n\t\t\t\tSystem.out.println(\"건너뜀: \" + candidate.label() + \" (\" + candidate.keyEnv() + \" 없음)\");\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\trunLlmStrategies(candidate.label(),\n\t\t\t\t\tmeters -> compatAdapter(candidate, apiKey, meters), candidate.cost(), cases, budget);\n\t\t}\n\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tif (bedrockModels != null && !bedrockModels.isBlank() && !budget.exhausted()) {\n\t\t\ttry (BedrockRuntimeClient bedrock = BedrockLlmAdapter.buildClient(\n\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"us-east-1\"))) {\n\t\t\t\tfor (String rawModelId : bedrockModels.split(\",\")) {\n\t\t\t\t\tif (budget.exhausted()) {\n\t\t\t\t\t\tprintBudgetStop(budget);\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\tString modelId = rawModelId.strip();\n\t\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId,\n\t\t\t\t\t\t\tmeters -> new BedrockLlmAdapter(bedrock, modelId, meters),\n\t\t\t\t\t\t\tnew ModelCost(1.0, 5.0), cases, budget);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tSystem.out.printf(\"누적 추정 비용: $%.4f / $%.2f%n\", budget.spentUsd(), budget.maxUsd());\n\t\tassertEquals(1.0, ruleResult.byCategory().get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.byCategory().get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\tprivate void runLlmStrategies(String modelLabel, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases, BenchBudget budget) {\n\t\tRunResult llm = runWithMeters(\"llm\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"llm\", llm);\n\t\tbudget.add(llm.costUsd());\n\t\tif (budget.exhausted()) {\n\t\t\treturn;\n\t\t}\n\n\t\tRunResult hybrid = runWithMeters(\"rule-first\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"rule-first\", hybrid);\n\t\tbudget.add(hybrid.costUsd());\n\t}\n\n\tprivate RunResult runWithMeters(String strategy, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases) {\n\t\tSimpleMeterRegistry meters = new SimpleMeterRegistry();\n\t\tLlmClaimExtractor llmExtractor = new LlmClaimExtractor(\n\t\t\t\tllmFactory.apply(meters), objectMapper, meters);\n\t\tExtractClaimsPort extractor = \"rule-first\".equals(strategy)\n\t\t\t\t? new RuleFirstClaimExtractor(new RuleClaimExtractor(), llmExtractor)\n\t\t\t\t: llmExtractor;\n\t\treturn run(strategy, extractor, cases, meters, cost);\n\t}\n\n\tprivate RunResult run(String strategy, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters, ModelCost cost) {\n\t\tlong startedAt = System.nanoTime();\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tTally row = new Tally((int) tp, fp, fn);\n\t\t\tbyCategory.merge(benchCase.category(), row,\n\t\t\t\t\t(left, right) -> left.plus(right.tp(), right.fp(), right.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tlong elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000;\n\t\tlong calls = meters == null ? 0 : Math.round(\n\t\t\t\tmeters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count());\n\t\tlong inputTokens = sumCounters(meters, \"baton.llm.tokens.input\");\n\t\tlong outputTokens = sumCounters(meters, \"baton.llm.tokens.output\");\n\t\tlong reasoningTokens = sumCounters(meters, \"baton.llm.tokens.reasoning\");\n\t\treturn new RunResult(Map.copyOf(byCategory), overall, calls, inputTokens, outputTokens,\n\t\t\t\treasoningTokens, cost.usd(inputTokens, outputTokens), elapsedMillis);\n\t}\n\n\tprivate LlmPort compatAdapter(CompatCandidate candidate, String apiKey, SimpleMeterRegistry meters) {\n\t\tString endpointPath = candidate.baseUrl().replaceAll(\"/+$\", \"\").endsWith(\"/v1\")\n\t\t\t\t? \"/chat/completions\" : \"/v1/chat/completions\";\n\t\tMap<String, Object> extraBody = candidate.thinking().isBlank()\n\t\t\t\t? Map.of() : Map.of(\"thinking\", Map.of(\"type\", candidate.thinking()));\n\t\tOpenAiCompatClient client = new OpenAiCompatClient(apiKey, candidate.model(),\n\t\t\t\tcandidate.baseUrl(), endpointPath, extraBody);\n\t\treturn new BenchCompatAdapter(candidate.label(), client, meters);\n\t}\n\n\tprivate List<CompatCandidate> compatCandidates() {\n\t\tString configured = System.getenv().getOrDefault(\"BENCH_COMPAT_MODELS\", DEFAULT_COMPAT_MODELS);\n\t\tList<CompatCandidate> candidates = new ArrayList<>();\n\t\tfor (String rawCandidate : configured.split(\";\")) {\n\t\t\tif (rawCandidate.isBlank()) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tString[] fields = rawCandidate.split(\"\\\\|\", -1);\n\t\t\tif (fields.length < 4 || fields.length > 5) {\n\t\t\t\tthrow new IllegalArgumentException(\n\t\t\t\t\t\t\"BENCH_COMPAT_MODELS 형식 오류: 라벨|base-url|키env|모델[|thinking]\");\n\t\t\t}\n\t\t\tString model = fields[3].strip();\n\t\t\tcandidates.add(new CompatCandidate(fields[0].strip(), fields[1].strip(), fields[2].strip(),\n\t\t\t\t\tmodel, fields.length == 5 ? fields[4].strip() : \"\", modelCost(model)));\n\t\t}\n\t\treturn List.copyOf(candidates);\n\t}\n\n\tprivate ModelCost modelCost(String model) {\n\t\tString normalized = model.toLowerCase();\n\t\tif (normalized.contains(\"kimi-k3\")) {\n\t\t\treturn new ModelCost(3.0, 15.0);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-flash\")) {\n\t\t\treturn new ModelCost(0.14, 0.28);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-pro\")) {\n\t\t\treturn new ModelCost(0.435, 0.87);\n\t\t}\n\t\tif (normalized.contains(\"solar-pro3\") || normalized.contains(\"gpt-4o-mini\")) {\n\t\t\treturn new ModelCost(0.15, 0.60);\n\t\t}\n\t\tthrow new IllegalArgumentException(\"벤치 단가 매핑이 없는 모델입니다: \" + model);\n\t}\n\n\tprivate void printHeader() {\n\t\tSystem.out.printf(\"%-28s %-12s %7s %7s %7s %7s %10s %10s %10s %9s %9s%n\",\n\t\t\t\t\"모델\", \"전략\", \"정밀도\", \"재현율\", \"F1\", \"호출\", \"입력토큰\", \"출력토큰\", \"추론토큰\", \"비용USD\", \"시간ms\");\n\t}\n\n\tprivate void printRow(String model, String strategy, RunResult result) {\n\t\tString verdict = \"rule\".equals(strategy) ? \"기준외\"\n\t\t\t\t: result.overall().f1() >= MIN_F1 ? \"통과\" : \"미달\";\n\t\tSystem.out.printf(\"%-28s %-12s %7.2f %7.2f %7.2f %7d %10d %10d %10d %9.4f %9d %s%n\",\n\t\t\t\tmodel, strategy, result.overall().precision(), result.overall().recall(),\n\t\t\t\tresult.overall().f1(), result.calls(), result.inputTokens(), result.outputTokens(),\n\t\t\t\tresult.reasoningTokens(), result.costUsd(), result.elapsedMillis(), verdict);\n\t}\n\n\tprivate void printBudgetStop(BenchBudget budget) {\n\t\tSystem.out.printf(\"중단: BENCH_MAX_USD 도달 ($%.4f / $%.2f), 남은 후보를 실행하지 않습니다.%n\",\n\t\t\t\tbudget.spentUsd(), budget.maxUsd());\n\t}\n\n\tprivate long sumCounters(SimpleMeterRegistry meters, String name) {\n\t\tif (meters == null) {\n\t\t\treturn 0;\n\t\t}\n\t\treturn Math.round(meters.find(name).counters().stream()\n\t\t\t\t.filter(counter -> counter.getId().getTag(\"provider\") != null)\n\t\t\t\t.mapToDouble(Counter::count).sum());\n\t}\n\n\tprivate double envDouble(String name, double defaultValue) {\n\t\tString value = System.getenv(name);\n\t\tdouble parsed = value == null || value.isBlank() ? defaultValue : Double.parseDouble(value);\n\t\tif (parsed < 0) {\n\t\t\tthrow new IllegalArgumentException(name + \"은 0 이상이어야 합니다\");\n\t\t}\n\t\treturn parsed;\n\t}\n\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> normalize = value -> value == null ? \"\" : value.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", normalize.apply(claim.subject()), normalize.apply(claim.attribute()),\n\t\t\t\tnormalize.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode claim = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", claim.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"attribute\").asString(\"\"), claim.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n\n\tprivate static final class BenchCompatAdapter extends OpenAiCompatibleLlmAdapter {\n\n\t\tprivate BenchCompatAdapter(String provider, OpenAiCompatClient client, SimpleMeterRegistry meters) {\n\t\t\tsuper(provider, client, LlmBudgetGuard.unlimited(meters));\n\t\t}\n\t}\n\n\tprivate static final class BenchBudget {\n\n\t\tprivate final double maxUsd;\n\t\tprivate double spentUsd;\n\n\t\tprivate BenchBudget(double maxUsd) {\n\t\t\tthis.maxUsd = maxUsd;\n\t\t}\n\n\t\tprivate void add(double usd) {\n\t\t\tspentUsd += usd;\n\t\t}\n\n\t\tprivate boolean exhausted() {\n\t\t\treturn spentUsd >= maxUsd;\n\t\t}\n\n\t\tprivate double spentUsd() {\n\t\t\treturn spentUsd;\n\t\t}\n\n\t\tprivate double maxUsd() {\n\t\t\treturn maxUsd;\n\t\t}\n\t}\n}\n"}, "58c30593f102df7bd833ad5525a56b39d3c0dc72": {"t": 1785755766.8183117, "n": 12975, "h": "5be4c4d591cff416", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmBudgetGuard;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiCompatClient;\nimport com.baton.adapter.out.llm.OpenAiCompatibleLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.application.port.out.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.Counter;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·비용·시간 비교.\n * 실행: ./gradlew benchTest. 실호출 후보는 키가 설정된 항목만 순차 실행한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\tprivate static final double MIN_F1 = 0.70;\n\tprivate static final String DEFAULT_COMPAT_MODELS = String.join(\";\",\n\t\t\t\"kimi|https://api.moonshot.ai/v1|MOONSHOT_API_KEY|kimi-k3\",\n\t\t\t\"deepseek|https://api.deepseek.com|DEEPSEEK_API_KEY|deepseek-v4-pro|disabled\",\n\t\t\t\"upstage|https://api.upstage.ai/v1|UPSTAGE_API_KEY|solar-pro3\");\n\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble precision = precision();\n\t\t\tdouble recall = recall();\n\t\t\treturn precision + recall == 0 ? 0.0 : 2 * precision * recall / (precision + recall);\n\t\t}\n\t}\n\n\trecord ModelCost(double inputPerMtok, double outputPerMtok) {\n\n\t\tdouble usd(long inputTokens, long outputTokens) {\n\t\t\treturn (inputTokens * inputPerMtok + outputTokens * outputPerMtok) / 1_000_000.0;\n\t\t}\n\t}\n\n\trecord CompatCandidate(String label, String baseUrl, String keyEnv, String model, String thinking,\n\t\t\tModelCost cost) {\n\t}\n\n\trecord RunResult(Map<String, Tally> byCategory, Tally overall, long calls,\n\t\t\tlong inputTokens, long outputTokens, long reasoningTokens, double costUsd, long elapsedMillis) {\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tdouble maxUsd = envDouble(\"BENCH_MAX_USD\", 2.0);\n\t\tBenchBudget budget = new BenchBudget(maxUsd);\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장, 지출 상한 $\"\n\t\t\t\t+ \"%.2f\".formatted(maxUsd) + \")\");\n\t\tprintHeader();\n\n\t\tRunResult ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()),\n\t\t\t\tcases, null, new ModelCost(0, 0));\n\t\tprintRow(\"-\", \"rule\", ruleResult);\n\n\t\tfor (CompatCandidate candidate : compatCandidates()) {\n\t\t\tif (budget.exhausted()) {\n\t\t\t\tprintBudgetStop(budget);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tString apiKey = System.getenv(candidate.keyEnv());\n\t\t\tif (apiKey == null || apiKey.isBlank()) {\n\t\t\t\tSystem.out.println(\"건너뜀: \" + candidate.label() + \" (\" + candidate.keyEnv() + \" 없음)\");\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\trunLlmStrategies(candidate.label(),\n\t\t\t\t\tmeters -> compatAdapter(candidate, apiKey, meters), candidate.cost(), cases, budget);\n\t\t}\n\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tif (bedrockModels != null && !bedrockModels.isBlank() && !budget.exhausted()) {\n\t\t\ttry (BedrockRuntimeClient bedrock = BedrockLlmAdapter.buildClient(\n\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"us-east-1\"))) {\n\t\t\t\tfor (String rawModelId : bedrockModels.split(\",\")) {\n\t\t\t\t\tif (budget.exhausted()) {\n\t\t\t\t\t\tprintBudgetStop(budget);\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\tString modelId = rawModelId.strip();\n\t\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId,\n\t\t\t\t\t\t\tmeters -> new BedrockLlmAdapter(bedrock, modelId, meters),\n\t\t\t\t\t\t\tnew ModelCost(1.0, 5.0), cases, budget);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tSystem.out.printf(\"누적 추정 비용: $%.4f / $%.2f%n\", budget.spentUsd(), budget.maxUsd());\n\t\tassertEquals(1.0, ruleResult.byCategory().get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.byCategory().get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\tprivate void runLlmStrategies(String modelLabel, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases, BenchBudget budget) {\n\t\tRunResult llm = runWithMeters(\"llm\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"llm\", llm);\n\t\tbudget.add(llm.costUsd());\n\t\tif (budget.exhausted()) {\n\t\t\treturn;\n\t\t}\n\n\t\tRunResult hybrid = runWithMeters(\"rule-first\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"rule-first\", hybrid);\n\t\tbudget.add(hybrid.costUsd());\n\t}\n\n\tprivate RunResult runWithMeters(String strategy, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases) {\n\t\tSimpleMeterRegistry meters = new SimpleMeterRegistry();\n\t\tLlmClaimExtractor llmExtractor = new LlmClaimExtractor(\n\t\t\t\tllmFactory.apply(meters), objectMapper, meters);\n\t\tExtractClaimsPort extractor = \"rule-first\".equals(strategy)\n\t\t\t\t? new RuleFirstClaimExtractor(new RuleClaimExtractor(), llmExtractor)\n\t\t\t\t: llmExtractor;\n\t\treturn run(strategy, extractor, cases, meters, cost);\n\t}\n\n\tprivate RunResult run(String strategy, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters, ModelCost cost) {\n\t\tlong startedAt = System.nanoTime();\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tTally row = new Tally((int) tp, fp, fn);\n\t\t\tbyCategory.merge(benchCase.category(), row,\n\t\t\t\t\t(left, right) -> left.plus(right.tp(), right.fp(), right.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tlong elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000;\n\t\tlong calls = meters == null ? 0 : Math.round(\n\t\t\t\tmeters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count());\n\t\tlong inputTokens = sumCounters(meters, \"baton.llm.tokens.input\");\n\t\tlong outputTokens = sumCounters(meters, \"baton.llm.tokens.output\");\n\t\tlong reasoningTokens = sumCounters(meters, \"baton.llm.tokens.reasoning\");\n\t\treturn new RunResult(Map.copyOf(byCategory), overall, calls, inputTokens, outputTokens,\n\t\t\t\treasoningTokens, cost.usd(inputTokens, outputTokens), elapsedMillis);\n\t}\n\n\tprivate LlmPort compatAdapter(CompatCandidate candidate, String apiKey, SimpleMeterRegistry meters) {\n\t\tString endpointPath = candidate.baseUrl().replaceAll(\"/+$\", \"\").endsWith(\"/v1\")\n\t\t\t\t? \"/chat/completions\" : \"/v1/chat/completions\";\n\t\tMap<String, Object> extraBody = candidate.thinking().isBlank()\n\t\t\t\t? Map.of() : Map.of(\"thinking\", Map.of(\"type\", candidate.thinking()));\n\t\tOpenAiCompatClient client = new OpenAiCompatClient(apiKey, candidate.model(),\n\t\t\t\tcandidate.baseUrl(), endpointPath, extraBody);\n\t\treturn new BenchCompatAdapter(candidate.label(), client, meters);\n\t}\n\n\tprivate List<CompatCandidate> compatCandidates() {\n\t\tString configured = System.getenv().getOrDefault(\"BENCH_COMPAT_MODELS\", DEFAULT_COMPAT_MODELS);\n\t\tList<CompatCandidate> candidates = new ArrayList<>();\n\t\tfor (String rawCandidate : configured.split(\";\")) {\n\t\t\tif (rawCandidate.isBlank()) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tString[] fields = rawCandidate.split(\"\\\\|\", -1);\n\t\t\tif (fields.length < 4 || fields.length > 5) {\n\t\t\t\tthrow new IllegalArgumentException(\n\t\t\t\t\t\t\"BENCH_COMPAT_MODELS 형식 오류: 라벨|base-url|키env|모델[|thinking]\");\n\t\t\t}\n\t\t\tString model = fields[3].strip();\n\t\t\tcandidates.add(new CompatCandidate(fields[0].strip(), fields[1].strip(), fields[2].strip(),\n\t\t\t\t\tmodel, fields.length == 5 ? fields[4].strip() : \"\", modelCost(model)));\n\t\t}\n\t\treturn List.copyOf(candidates);\n\t}\n\n\tprivate ModelCost modelCost(String model) {\n\t\tString normalized = model.toLowerCase();\n\t\tif (normalized.contains(\"kimi-k3\")) {\n\t\t\treturn new ModelCost(3.0, 15.0);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-flash\")) {\n\t\t\treturn new ModelCost(0.14, 0.28);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-pro\")) {\n\t\t\treturn new ModelCost(0.435, 0.87);\n\t\t}\n\t\tif (normalized.contains(\"solar-pro3\") || normalized.contains(\"gpt-4o-mini\")) {\n\t\t\treturn new ModelCost(0.15, 0.60);\n\t\t}\n\t\tthrow new IllegalArgumentException(\"벤치 단가 매핑이 없는 모델입니다: \" + model);\n\t}\n\n\tprivate void printHeader() {\n\t\tSystem.out.printf(\"%-28s %-12s %7s %7s %7s %7s %10s %10s %10s %9s %9s%n\",\n\t\t\t\t\"모델\", \"전략\", \"정밀도\", \"재현율\", \"F1\", \"호출\", \"입력토큰\", \"출력토큰\", \"추론토큰\", \"비용USD\", \"시간ms\");\n\t}\n\n\tprivate void printRow(String model, String strategy, RunResult result) {\n\t\tString verdict = \"rule\".equals(strategy) ? \"기준외\"\n\t\t\t\t: result.overall().f1() >= MIN_F1 ? \"통과\" : \"미달\";\n\t\tSystem.out.printf(\"%-28s %-12s %7.2f %7.2f %7.2f %7d %10d %10d %10d %9.4f %9d %s%n\",\n\t\t\t\tmodel, strategy, result.overall().precision(), result.overall().recall(),\n\t\t\t\tresult.overall().f1(), result.calls(), result.inputTokens(), result.outputTokens(),\n\t\t\t\tresult.reasoningTokens(), result.costUsd(), result.elapsedMillis(), verdict);\n\t}\n\n\tprivate void printBudgetStop(BenchBudget budget) {\n\t\tSystem.out.printf(\"중단: BENCH_MAX_USD 도달 ($%.4f / $%.2f), 남은 후보를 실행하지 않습니다.%n\",\n\t\t\t\tbudget.spentUsd(), budget.maxUsd());\n\t}\n\n\tprivate long sumCounters(SimpleMeterRegistry meters, String name) {\n\t\tif (meters == null) {\n\t\t\treturn 0;\n\t\t}\n\t\treturn Math.round(meters.find(name).counters().stream()\n\t\t\t\t.filter(counter -> counter.getId().getTag(\"provider\") != null)\n\t\t\t\t.mapToDouble(Counter::count).sum());\n\t}\n\n\tprivate double envDouble(String name, double defaultValue) {\n\t\tString value = System.getenv(name);\n\t\tdouble parsed = value == null || value.isBlank() ? defaultValue : Double.parseDouble(value);\n\t\tif (parsed < 0) {\n\t\t\tthrow new IllegalArgumentException(name + \"은 0 이상이어야 합니다\");\n\t\t}\n\t\treturn parsed;\n\t}\n\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> normalize = value -> value == null ? \"\" : value.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", normalize.apply(claim.subject()), normalize.apply(claim.attribute()),\n\t\t\t\tnormalize.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode claim = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", claim.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"attribute\").asString(\"\"), claim.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n\n\tprivate static final class BenchCompatAdapter extends OpenAiCompatibleLlmAdapter {\n\n\t\tprivate BenchCompatAdapter(String provider, OpenAiCompatClient client, SimpleMeterRegistry meters) {\n\t\t\tsuper(provider, client, LlmBudgetGuard.unlimited(meters));\n\t\t}\n\t}\n\n\tprivate static final class BenchBudget {\n\n\t\tprivate final double maxUsd;\n\t\tprivate double spentUsd;\n\n\t\tprivate BenchBudget(double maxUsd) {\n\t\t\tthis.maxUsd = maxUsd;\n\t\t}\n\n\t\tprivate void add(double usd) {\n\t\t\tspentUsd += usd;\n\t\t}\n\n\t\tprivate boolean exhausted() {\n\t\t\treturn spentUsd >= maxUsd;\n\t\t}\n\n\t\tprivate double spentUsd() {\n\t\t\treturn spentUsd;\n\t\t}\n\n\t\tprivate double maxUsd() {\n\t\t\treturn maxUsd;\n\t\t}\n\t}\n}\n"}, "55769e6e9e607e931c69d492fb5115ff382469d5": {"t": 1785755768.437406, "n": 12903, "h": "ce418d9b7a585d64", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmBudgetGuard;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiCompatClient;\nimport com.baton.adapter.out.llm.OpenAiCompatibleLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.application.port.out.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.Counter;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·비용·시간 비교.\n * 실행: ./gradlew benchTest. 실호출 후보는 키가 설정된 항목만 순차 실행한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\tprivate static final double MIN_F1 = 0.70;\n\tprivate static final String DEFAULT_COMPAT_MODELS = String.join(\";\",\n\t\t\t\"kimi|https://api.moonshot.ai/v1|MOONSHOT_API_KEY|kimi-k3\",\n\t\t\t\"deepseek|https://api.deepseek.com|DEEPSEEK_API_KEY|deepseek-v4-pro|disabled\",\n\t\t\t\"upstage|https://api.upstage.ai/v1|UPSTAGE_API_KEY|solar-pro3\");\n\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble precision = precision();\n\t\t\tdouble recall = recall();\n\t\t\treturn precision + recall == 0 ? 0.0 : 2 * precision * recall / (precision + recall);\n\t\t}\n\t}\n\n\trecord ModelCost(double inputPerMtok, double outputPerMtok) {\n\n\t\tdouble usd(long inputTokens, long outputTokens) {\n\t\t\treturn (inputTokens * inputPerMtok + outputTokens * outputPerMtok) / 1_000_000.0;\n\t\t}\n\t}\n\n\trecord CompatCandidate(String label, String baseUrl, String keyEnv, String model, String thinking,\n\t\t\tModelCost cost) {\n\t}\n\n\trecord RunResult(Map<String, Tally> byCategory, Tally overall, long calls,\n\t\t\tlong inputTokens, long outputTokens, long reasoningTokens, double costUsd, long elapsedMillis) {\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tdouble maxUsd = envDouble(\"BENCH_MAX_USD\", 2.0);\n\t\tBenchBudget budget = new BenchBudget(maxUsd);\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장, 지출 상한 $\"\n\t\t\t\t+ \"%.2f\".formatted(maxUsd) + \")\");\n\t\tprintHeader();\n\n\t\tRunResult ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()),\n\t\t\t\tcases, null, new ModelCost(0, 0));\n\t\tprintRow(\"-\", \"rule\", ruleResult);\n\n\t\tfor (CompatCandidate candidate : compatCandidates()) {\n\t\t\tif (budget.exhausted()) {\n\t\t\t\tprintBudgetStop(budget);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tString apiKey = System.getenv(candidate.keyEnv());\n\t\t\tif (apiKey == null || apiKey.isBlank()) {\n\t\t\t\tSystem.out.println(\"건너뜀: \" + candidate.label() + \" (\" + candidate.keyEnv() + \" 없음)\");\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\trunLlmStrategies(candidate.label(),\n\t\t\t\t\tmeters -> compatAdapter(candidate, apiKey, meters), candidate.cost(), cases, budget);\n\t\t}\n\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tif (bedrockModels != null && !bedrockModels.isBlank() && !budget.exhausted()) {\n\t\t\ttry (BedrockRuntimeClient bedrock = BedrockLlmAdapter.buildClient(\n\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"us-east-1\"))) {\n\t\t\t\tfor (String rawModelId : bedrockModels.split(\",\")) {\n\t\t\t\t\tif (budget.exhausted()) {\n\t\t\t\t\t\tprintBudgetStop(budget);\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\tString modelId = rawModelId.strip();\n\t\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId,\n\t\t\t\t\t\t\tmeters -> new BedrockLlmAdapter(bedrock, modelId, meters),\n\t\t\t\t\t\t\tnew ModelCost(1.0, 5.0), cases, budget);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tSystem.out.printf(\"누적 추정 비용: $%.4f / $%.2f%n\", budget.spentUsd(), budget.maxUsd());\n\t\tassertEquals(1.0, ruleResult.byCategory().get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.byCategory().get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\tprivate void runLlmStrategies(String modelLabel, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases, BenchBudget budget) {\n\t\tRunResult llm = runWithMeters(\"llm\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"llm\", llm);\n\t\tbudget.add(llm.costUsd());\n\t\tif (budget.exhausted()) {\n\t\t\treturn;\n\t\t}\n\n\t\tRunResult hybrid = runWithMeters(\"rule-first\", llmFactory, cost, cases);\n\t\tprintRow(modelLabel, \"rule-first\", hybrid);\n\t\tbudget.add(hybrid.costUsd());\n\t}\n\n\tprivate RunResult runWithMeters(String strategy, Function<SimpleMeterRegistry, LlmPort> llmFactory,\n\t\t\tModelCost cost, List<BenchCase> cases) {\n\t\tSimpleMeterRegistry meters = new SimpleMeterRegistry();\n\t\tLlmClaimExtractor llmExtractor = new LlmClaimExtractor(\n\t\t\t\tllmFactory.apply(meters), objectMapper, meters);\n\t\tExtractClaimsPort extractor = \"rule-first\".equals(strategy)\n\t\t\t\t? new RuleFirstClaimExtractor(new RuleClaimExtractor(), llmExtractor)\n\t\t\t\t: llmExtractor;\n\t\treturn run(strategy, extractor, cases, meters, cost);\n\t}\n\n\tprivate RunResult run(String strategy, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters, ModelCost cost) {\n\t\tlong startedAt = System.nanoTime();\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tTally row = new Tally((int) tp, fp, fn);\n\t\t\tbyCategory.merge(benchCase.category(), row,\n\t\t\t\t\t(left, right) -> left.plus(right.tp(), right.fp(), right.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tlong elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000;\n\t\tlong calls = meters == null ? 0 : Math.round(\n\t\t\t\tmeters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count());\n\t\tlong inputTokens = sumCounters(meters, \"baton.llm.tokens.input\");\n\t\tlong outputTokens = sumCounters(meters, \"baton.llm.tokens.output\");\n\t\tlong reasoningTokens = sumCounters(meters, \"baton.llm.tokens.reasoning\");\n\t\treturn new RunResult(Map.copyOf(byCategory), overall, calls, inputTokens, outputTokens,\n\t\t\t\treasoningTokens, cost.usd(inputTokens, outputTokens), elapsedMillis);\n\t}\n\n\tprivate LlmPort compatAdapter(CompatCandidate candidate, String apiKey, SimpleMeterRegistry meters) {\n\t\tString endpointPath = candidate.baseUrl().replaceAll(\"/+$\", \"\").endsWith(\"/v1\")\n\t\t\t\t? \"/chat/completions\" : \"/v1/chat/completions\";\n\t\tMap<String, Object> extraBody = candidate.thinking().isBlank()\n\t\t\t\t? Map.of() : Map.of(\"thinking\", Map.of(\"type\", candidate.thinking()));\n\t\tOpenAiCompatClient client = new OpenAiCompatClient(apiKey, candidate.model(),\n\t\t\t\tcandidate.baseUrl(), endpointPath, extraBody);\n\t\treturn new BenchCompatAdapter(candidate.label(), client, meters);\n\t}\n\n\tprivate List<CompatCandidate> compatCandidates() {\n\t\tString configured = System.getenv().getOrDefault(\"BENCH_COMPAT_MODELS\", DEFAULT_COMPAT_MODELS);\n\t\tList<CompatCandidate> candidates = new ArrayList<>();\n\t\tfor (String rawCandidate : configured.split(\";\")) {\n\t\t\tif (rawCandidate.isBlank()) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tString[] fields = rawCandidate.split(\"\\\\|\", -1);\n\t\t\tif (fields.length < 4 || fields.length > 5) {\n\t\t\t\tthrow new IllegalArgumentException(\n\t\t\t\t\t\t\"BENCH_COMPAT_MODELS 형식 오류: 라벨|base-url|키env|모델[|thinking]\");\n\t\t\t}\n\t\t\tString model = fields[3].strip();\n\t\t\tcandidates.add(new CompatCandidate(fields[0].strip(), fields[1].strip(), fields[2].strip(),\n\t\t\t\t\tmodel, fields.length == 5 ? fields[4].strip() : \"\", modelCost(model)));\n\t\t}\n\t\treturn List.copyOf(candidates);\n\t}\n\n\tprivate ModelCost modelCost(String model) {\n\t\tString normalized = model.toLowerCase();\n\t\tif (normalized.contains(\"kimi-k3\")) {\n\t\t\treturn new ModelCost(3.0, 15.0);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-flash\")) {\n\t\t\treturn new ModelCost(0.14, 0.28);\n\t\t}\n\t\tif (normalized.contains(\"deepseek-v4-pro\")) {\n\t\t\treturn new ModelCost(0.435, 0.87);\n\t\t}\n\t\tif (normalized.contains(\"solar-pro3\") || normalized.contains(\"gpt-4o-mini\")) {\n\t\t\treturn new ModelCost(0.15, 0.60);\n\t\t}\n\t\tthrow new IllegalArgumentException(\"벤치 단가 매핑이 없는 모델입니다: \" + model);\n\t}\n\n\tprivate void printHeader() {\n\t\tSystem.out.printf(\"%-28s %-12s %7s %7s %7s %7s %10s %10s %10s %9s %9s%n\",\n\t\t\t\t\"모델\", \"전략\", \"정밀도\", \"재현율\", \"F1\", \"호출\", \"입력토큰\", \"출력토큰\", \"추론토큰\", \"비용USD\", \"시간ms\");\n\t}\n\n\tprivate void printRow(String model, String strategy, RunResult result) {\n\t\tString verdict = \"rule\".equals(strategy) ? \"기준외\"\n\t\t\t\t: result.overall().f1() >= MIN_F1 ? \"통과\" : \"미달\";\n\t\tSystem.out.printf(\"%-28s %-12s %7.2f %7.2f %7.2f %7d %10d %10d %10d %9.4f %9d %s%n\",\n\t\t\t\tmodel, strategy, result.overall().precision(), result.overall().recall(),\n\t\t\t\tresult.overall().f1(), result.calls(), result.inputTokens(), result.outputTokens(),\n\t\t\t\tresult.reasoningTokens(), result.costUsd(), result.elapsedMillis(), verdict);\n\t}\n\n\tprivate void printBudgetStop(BenchBudget budget) {\n\t\tSystem.out.printf(\"중단: BENCH_MAX_USD 도달 ($%.4f / $%.2f), 남은 후보를 실행하지 않습니다.%n\",\n\t\t\t\tbudget.spentUsd(), budget.maxUsd());\n\t}\n\n\tprivate long sumCounters(SimpleMeterRegistry meters, String name) {\n\t\tif (meters == null) {\n\t\t\treturn 0;\n\t\t}\n\t\treturn Math.round(meters.find(name).counters().stream().mapToDouble(Counter::count).sum());\n\t}\n\n\tprivate double envDouble(String name, double defaultValue) {\n\t\tString value = System.getenv(name);\n\t\tdouble parsed = value == null || value.isBlank() ? defaultValue : Double.parseDouble(value);\n\t\tif (parsed < 0) {\n\t\t\tthrow new IllegalArgumentException(name + \"은 0 이상이어야 합니다\");\n\t\t}\n\t\treturn parsed;\n\t}\n\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> normalize = value -> value == null ? \"\" : value.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", normalize.apply(claim.subject()), normalize.apply(claim.attribute()),\n\t\t\t\tnormalize.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode claim = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", claim.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"attribute\").asString(\"\"), claim.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tclaim.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n\n\tprivate static final class BenchCompatAdapter extends OpenAiCompatibleLlmAdapter {\n\n\t\tprivate BenchCompatAdapter(String provider, OpenAiCompatClient client, SimpleMeterRegistry meters) {\n\t\t\tsuper(provider, client, LlmBudgetGuard.unlimited(meters));\n\t\t}\n\t}\n\n\tprivate static final class BenchBudget {\n\n\t\tprivate final double maxUsd;\n\t\tprivate double spentUsd;\n\n\t\tprivate BenchBudget(double maxUsd) {\n\t\t\tthis.maxUsd = maxUsd;\n\t\t}\n\n\t\tprivate void add(double usd) {\n\t\t\tspentUsd += usd;\n\t\t}\n\n\t\tprivate boolean exhausted() {\n\t\t\treturn spentUsd >= maxUsd;\n\t\t}\n\n\t\tprivate double spentUsd() {\n\t\t\treturn spentUsd;\n\t\t}\n\n\t\tprivate double maxUsd() {\n\t\t\treturn maxUsd;\n\t\t}\n\t}\n}\n"}, "1cea12e3dcc3eabcf74a40f4829444c56fa29e10": {"t": 1785755769.869965, "n": 7630, "h": "8a306285afc8ca5a", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.application.port.out.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·LLM 호출 수 비교.\n * 골든 셋: claim-bench/cases.jsonl (한국어 업무 문장 + 수작업 라벨).\n * 실행: ./gradlew benchTest (평소 test 태스크에선 제외 — @Tag(\"bench\"))\n *\n * 실 LLM 측정(있는 것만 돌고, 없으면 rule만):\n * - Bedrock(선택 비교군): BENCH_BEDROCK_MODELS=모델ID1,모델ID2 … (+ AWS 자격증명, 기본 us-east-1)\n * → 모델별로 llm·rule-first를 각각 측정해 최적 모델·전략을 고른다.\n * - OpenAI(비교용): OPENAI_API_KEY (+ BENCH_LLM_MODEL 기본 gpt-4o-mini)\n * 기본 전략(app.claim.extractor)·기본 모델 확정은 이 결과로 한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\t// context = 원문 전체(스레드·문서). null이면 문장 = 원문(파이프라인의 단문 문서와 동일)\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble p = precision(), r = recall();\n\t\t\treturn p + r == 0 ? 0.0 : 2 * p * r / (p + r);\n\t\t}\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tString openAiKey = System.getenv(\"OPENAI_API_KEY\");\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tboolean openAiAvailable = openAiKey != null && !openAiKey.isBlank();\n\t\tboolean bedrockAvailable = bedrockModels != null && !bedrockModels.isBlank();\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장: template/freeform/keyvalue/qna/noise)\");\n\t\tif (!openAiAvailable && !bedrockAvailable) {\n\t\t\tSystem.out.println(\"실 LLM 설정 없음 — rule 전략만 측정 \"\n\t\t\t\t\t+ \"(BENCH_BEDROCK_MODELS + AWS 자격증명 또는 OPENAI_API_KEY 설정 후 재실행)\");\n\t\t}\n\t\tSystem.out.println();\n\t\tSystem.out.printf(\"%-12s %-10s %8s %8s %8s %10s%n\", \"전략\", \"구간\", \"정밀도\", \"재현율\", \"F1\", \"LLM호출\");\n\n\t\tMap<String, Tally> ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()), cases, null);\n\n\t\tif (bedrockAvailable) {\n\t\t\tBedrockRuntimeClient bedrock = BedrockLlmAdapter.buildClient(\n\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"us-east-1\"));\n\t\t\tfor (String modelId : bedrockModels.split(\",\")) {\n\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId.strip(),\n\t\t\t\t\t\tnew BedrockLlmAdapter(bedrock, modelId.strip()), cases);\n\t\t\t}\n\t\t}\n\t\tif (openAiAvailable) {\n\t\t\trunLlmStrategies(\"openai:\" + System.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\tnew OpenAiLlmAdapter(openAiKey,\n\t\t\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\t\t\t\"https://api.openai.com\"),\n\t\t\t\t\tcases);\n\t\t}\n\n\t\t// 룰은 결정적 — 정형 문장 전부 추출(재현율 1.0), 노이즈에서 오탐 0이어야 한다\n\t\tassertEquals(1.0, ruleResult.get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\t/** 모델 하나에 대해 llm(단독)·rule-first(폴백) 두 전략을 측정한다. */\n\tprivate void runLlmStrategies(String modelLabel, LlmPort llm, List<BenchCase> cases) {\n\t\tSystem.out.println(\"### 모델: \" + modelLabel);\n\t\tSimpleMeterRegistry llmMeters = new SimpleMeterRegistry();\n\t\trun(\"llm\", new LlmClaimExtractor(llm, objectMapper, llmMeters), cases, llmMeters);\n\n\t\tSimpleMeterRegistry hybridMeters = new SimpleMeterRegistry();\n\t\trun(\"rule-first\", new RuleFirstClaimExtractor(new RuleClaimExtractor(),\n\t\t\t\tnew LlmClaimExtractor(llm, objectMapper, hybridMeters)), cases, hybridMeters);\n\t}\n\n\tprivate Map<String, Tally> run(String name, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters) {\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\t// 파이프라인과 동일하게 룰필터를 먼저 통과시킨다\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tbyCategory.merge(benchCase.category(), new Tally((int) tp, fp, fn),\n\t\t\t\t\t(a, b) -> a.plus(b.tp(), b.fp(), b.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tdouble llmCalls = meters == null ? 0\n\t\t\t\t: meters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count();\n\t\tfor (Map.Entry<String, Tally> entry : byCategory.entrySet()) {\n\t\t\tprintRow(name, entry.getKey(), entry.getValue(), null);\n\t\t}\n\t\tprintRow(name, \"전체\", overall, llmCalls);\n\t\tSystem.out.println();\n\t\treturn byCategory;\n\t}\n\n\tprivate void printRow(String strategy, String segment, Tally tally, Double llmCalls) {\n\t\tSystem.out.printf(\"%-12s %-10s %8.2f %8.2f %8.2f %10s%n\", strategy, segment,\n\t\t\t\ttally.precision(), tally.recall(), tally.f1(), llmCalls == null ? \"-\" : \"%.0f\".formatted(llmCalls));\n\t}\n\n\t/** 채점 키 — 주어·속성·값·극성(시점·범위는 채점 제외). 공백 정규화. */\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> norm = s -> s == null ? \"\" : s.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", norm.apply(claim.subject()), norm.apply(claim.attribute()),\n\t\t\t\tnorm.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode c = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", c.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"attribute\").asString(\"\"), c.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n}\n"}, "56be88f66e022391cfc1b543330fbcac0684a484": {"t": 1785755771.3014817, "n": 7659, "h": "a0a6e8b8c0a96866", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.application.port.out.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·LLM 호출 수 비교.\n * 골든 셋: claim-bench/cases.jsonl (한국어 업무 문장 + 수작업 라벨).\n * 실행: ./gradlew benchTest (평소 test 태스크에선 제외 — @Tag(\"bench\"))\n *\n * 실 LLM 측정(있는 것만 돌고, 없으면 rule만):\n * - Bedrock(팀 확정): BENCH_BEDROCK_MODELS=모델ID1,모델ID2 … (+ AWS 자격증명, BENCH_BEDROCK_REGION 기본 ap-northeast-2)\n * → 모델별로 llm·rule-first를 각각 측정해 최적 모델·전략을 고른다.\n * - OpenAI(비교용): OPENAI_API_KEY (+ BENCH_LLM_MODEL 기본 gpt-4o-mini)\n * 기본 전략(app.claim.extractor)·기본 모델 확정은 이 결과로 한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\t// context = 원문 전체(스레드·문서). null이면 문장 = 원문(파이프라인의 단문 문서와 동일)\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble p = precision(), r = recall();\n\t\t\treturn p + r == 0 ? 0.0 : 2 * p * r / (p + r);\n\t\t}\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tString openAiKey = System.getenv(\"OPENAI_API_KEY\");\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tboolean openAiAvailable = openAiKey != null && !openAiKey.isBlank();\n\t\tboolean bedrockAvailable = bedrockModels != null && !bedrockModels.isBlank();\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장: template/freeform/keyvalue/qna/noise)\");\n\t\tif (!openAiAvailable && !bedrockAvailable) {\n\t\t\tSystem.out.println(\"실 LLM 설정 없음 — rule 전략만 측정 \"\n\t\t\t\t\t+ \"(BENCH_BEDROCK_MODELS + AWS 자격증명 또는 OPENAI_API_KEY 설정 후 재실행)\");\n\t\t}\n\t\tSystem.out.println();\n\t\tSystem.out.printf(\"%-12s %-10s %8s %8s %8s %10s%n\", \"전략\", \"구간\", \"정밀도\", \"재현율\", \"F1\", \"LLM호출\");\n\n\t\tMap<String, Tally> ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()), cases, null);\n\n\t\tif (bedrockAvailable) {\n\t\t\tBedrockRuntimeClient bedrock = BedrockLlmAdapter.buildClient(\n\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"ap-northeast-2\"));\n\t\t\tfor (String modelId : bedrockModels.split(\",\")) {\n\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId.strip(),\n\t\t\t\t\t\tnew BedrockLlmAdapter(bedrock, modelId.strip()), cases);\n\t\t\t}\n\t\t}\n\t\tif (openAiAvailable) {\n\t\t\trunLlmStrategies(\"openai:\" + System.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\tnew OpenAiLlmAdapter(openAiKey,\n\t\t\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\t\t\t\"https://api.openai.com\"),\n\t\t\t\t\tcases);\n\t\t}\n\n\t\t// 룰은 결정적 — 정형 문장 전부 추출(재현율 1.0), 노이즈에서 오탐 0이어야 한다\n\t\tassertEquals(1.0, ruleResult.get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\t/** 모델 하나에 대해 llm(단독)·rule-first(폴백) 두 전략을 측정한다. */\n\tprivate void runLlmStrategies(String modelLabel, LlmPort llm, List<BenchCase> cases) {\n\t\tSystem.out.println(\"### 모델: \" + modelLabel);\n\t\tSimpleMeterRegistry llmMeters = new SimpleMeterRegistry();\n\t\trun(\"llm\", new LlmClaimExtractor(llm, objectMapper, llmMeters), cases, llmMeters);\n\n\t\tSimpleMeterRegistry hybridMeters = new SimpleMeterRegistry();\n\t\trun(\"rule-first\", new RuleFirstClaimExtractor(new RuleClaimExtractor(),\n\t\t\t\tnew LlmClaimExtractor(llm, objectMapper, hybridMeters)), cases, hybridMeters);\n\t}\n\n\tprivate Map<String, Tally> run(String name, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters) {\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\t// 파이프라인과 동일하게 룰필터를 먼저 통과시킨다\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tbyCategory.merge(benchCase.category(), new Tally((int) tp, fp, fn),\n\t\t\t\t\t(a, b) -> a.plus(b.tp(), b.fp(), b.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tdouble llmCalls = meters == null ? 0\n\t\t\t\t: meters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count();\n\t\tfor (Map.Entry<String, Tally> entry : byCategory.entrySet()) {\n\t\t\tprintRow(name, entry.getKey(), entry.getValue(), null);\n\t\t}\n\t\tprintRow(name, \"전체\", overall, llmCalls);\n\t\tSystem.out.println();\n\t\treturn byCategory;\n\t}\n\n\tprivate void printRow(String strategy, String segment, Tally tally, Double llmCalls) {\n\t\tSystem.out.printf(\"%-12s %-10s %8.2f %8.2f %8.2f %10s%n\", strategy, segment,\n\t\t\t\ttally.precision(), tally.recall(), tally.f1(), llmCalls == null ? \"-\" : \"%.0f\".formatted(llmCalls));\n\t}\n\n\t/** 채점 키 — 주어·속성·값·극성(시점·범위는 채점 제외). 공백 정규화. */\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> norm = s -> s == null ? \"\" : s.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", norm.apply(claim.subject()), norm.apply(claim.attribute()),\n\t\t\t\tnorm.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode c = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", c.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"attribute\").asString(\"\"), c.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n}\n"}, "ba67cefbf4e14942fa240e8703c958e17608d702": {"t": 1785755772.5023263, "n": 7738, "h": "861461d93125b85f", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.application.port.out.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.regions.Region;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·LLM 호출 수 비교.\n * 골든 셋: claim-bench/cases.jsonl (한국어 업무 문장 + 수작업 라벨).\n * 실행: ./gradlew benchTest (평소 test 태스크에선 제외 — @Tag(\"bench\"))\n *\n * 실 LLM 측정(있는 것만 돌고, 없으면 rule만):\n * - Bedrock(팀 확정): BENCH_BEDROCK_MODELS=모델ID1,모델ID2 … (+ AWS 자격증명, BENCH_BEDROCK_REGION 기본 ap-northeast-2)\n * → 모델별로 llm·rule-first를 각각 측정해 최적 모델·전략을 고른다.\n * - OpenAI(비교용): OPENAI_API_KEY (+ BENCH_LLM_MODEL 기본 gpt-4o-mini)\n * 기본 전략(app.claim.extractor)·기본 모델 확정은 이 결과로 한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\t// context = 원문 전체(스레드·문서). null이면 문장 = 원문(파이프라인의 단문 문서와 동일)\n\trecord BenchCase(String category, String sentence, String context, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble p = precision(), r = recall();\n\t\t\treturn p + r == 0 ? 0.0 : 2 * p * r / (p + r);\n\t\t}\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tString openAiKey = System.getenv(\"OPENAI_API_KEY\");\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tboolean openAiAvailable = openAiKey != null && !openAiKey.isBlank();\n\t\tboolean bedrockAvailable = bedrockModels != null && !bedrockModels.isBlank();\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장: template/freeform/keyvalue/qna/noise)\");\n\t\tif (!openAiAvailable && !bedrockAvailable) {\n\t\t\tSystem.out.println(\"실 LLM 설정 없음 — rule 전략만 측정 \"\n\t\t\t\t\t+ \"(BENCH_BEDROCK_MODELS + AWS 자격증명 또는 OPENAI_API_KEY 설정 후 재실행)\");\n\t\t}\n\t\tSystem.out.println();\n\t\tSystem.out.printf(\"%-12s %-10s %8s %8s %8s %10s%n\", \"전략\", \"구간\", \"정밀도\", \"재현율\", \"F1\", \"LLM호출\");\n\n\t\tMap<String, Tally> ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()), cases, null);\n\n\t\tif (bedrockAvailable) {\n\t\t\tBedrockRuntimeClient bedrock = BedrockRuntimeClient.builder()\n\t\t\t\t\t.region(Region.of(System.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"ap-northeast-2\")))\n\t\t\t\t\t.build();\n\t\t\tfor (String modelId : bedrockModels.split(\",\")) {\n\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId.strip(),\n\t\t\t\t\t\tnew BedrockLlmAdapter(bedrock, modelId.strip()), cases);\n\t\t\t}\n\t\t}\n\t\tif (openAiAvailable) {\n\t\t\trunLlmStrategies(\"openai:\" + System.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\tnew OpenAiLlmAdapter(openAiKey,\n\t\t\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\t\t\t\"https://api.openai.com\"),\n\t\t\t\t\tcases);\n\t\t}\n\n\t\t// 룰은 결정적 — 정형 문장 전부 추출(재현율 1.0), 노이즈에서 오탐 0이어야 한다\n\t\tassertEquals(1.0, ruleResult.get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\t/** 모델 하나에 대해 llm(단독)·rule-first(폴백) 두 전략을 측정한다. */\n\tprivate void runLlmStrategies(String modelLabel, LlmPort llm, List<BenchCase> cases) {\n\t\tSystem.out.println(\"### 모델: \" + modelLabel);\n\t\tSimpleMeterRegistry llmMeters = new SimpleMeterRegistry();\n\t\trun(\"llm\", new LlmClaimExtractor(llm, objectMapper, llmMeters), cases, llmMeters);\n\n\t\tSimpleMeterRegistry hybridMeters = new SimpleMeterRegistry();\n\t\trun(\"rule-first\", new RuleFirstClaimExtractor(new RuleClaimExtractor(),\n\t\t\t\tnew LlmClaimExtractor(llm, objectMapper, hybridMeters)), cases, hybridMeters);\n\t}\n\n\tprivate Map<String, Tally> run(String name, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters) {\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\t// 파이프라인과 동일하게 룰필터를 먼저 통과시킨다\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence()), benchCase.context()) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tbyCategory.merge(benchCase.category(), new Tally((int) tp, fp, fn),\n\t\t\t\t\t(a, b) -> a.plus(b.tp(), b.fp(), b.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tdouble llmCalls = meters == null ? 0\n\t\t\t\t: meters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count();\n\t\tfor (Map.Entry<String, Tally> entry : byCategory.entrySet()) {\n\t\t\tprintRow(name, entry.getKey(), entry.getValue(), null);\n\t\t}\n\t\tprintRow(name, \"전체\", overall, llmCalls);\n\t\tSystem.out.println();\n\t\treturn byCategory;\n\t}\n\n\tprivate void printRow(String strategy, String segment, Tally tally, Double llmCalls) {\n\t\tSystem.out.printf(\"%-12s %-10s %8.2f %8.2f %8.2f %10s%n\", strategy, segment,\n\t\t\t\ttally.precision(), tally.recall(), tally.f1(), llmCalls == null ? \"-\" : \"%.0f\".formatted(llmCalls));\n\t}\n\n\t/** 채점 키 — 주어·속성·값·극성(시점·범위는 채점 제외). 공백 정규화. */\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> norm = s -> s == null ? \"\" : s.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", norm.apply(claim.subject()), norm.apply(claim.attribute()),\n\t\t\t\tnorm.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode c = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", c.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"attribute\").asString(\"\"), c.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), node.path(\"context\").asString(null), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n}\n"}, "394ebf4588d96c83eb0179e7d93d1f7b87f31a1e": {"t": 1785755773.9061182, "n": 7589, "h": "20433af0f994e7e9", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.BedrockLlmAdapter;\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.application.port.out.LlmPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport software.amazon.awssdk.regions.Region;\nimport software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first) × 모델별 정밀도·재현율·F1·LLM 호출 수 비교.\n * 골든 셋: claim-bench/cases.jsonl (한국어 업무 문장 + 수작업 라벨).\n * 실행: ./gradlew benchTest (평소 test 태스크에선 제외 — @Tag(\"bench\"))\n *\n * 실 LLM 측정(있는 것만 돌고, 없으면 rule만):\n * - Bedrock(팀 확정): BENCH_BEDROCK_MODELS=모델ID1,모델ID2 … (+ AWS 자격증명, BENCH_BEDROCK_REGION 기본 ap-northeast-2)\n * → 모델별로 llm·rule-first를 각각 측정해 최적 모델·전략을 고른다.\n * - OpenAI(비교용): OPENAI_API_KEY (+ BENCH_LLM_MODEL 기본 gpt-4o-mini)\n * 기본 전략(app.claim.extractor)·기본 모델 확정은 이 결과로 한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\trecord BenchCase(String category, String sentence, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble p = precision(), r = recall();\n\t\t\treturn p + r == 0 ? 0.0 : 2 * p * r / (p + r);\n\t\t}\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tString openAiKey = System.getenv(\"OPENAI_API_KEY\");\n\t\tString bedrockModels = System.getenv(\"BENCH_BEDROCK_MODELS\");\n\t\tboolean openAiAvailable = openAiKey != null && !openAiKey.isBlank();\n\t\tboolean bedrockAvailable = bedrockModels != null && !bedrockModels.isBlank();\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장: template/freeform/noise)\");\n\t\tif (!openAiAvailable && !bedrockAvailable) {\n\t\t\tSystem.out.println(\"실 LLM 설정 없음 — rule 전략만 측정 \"\n\t\t\t\t\t+ \"(BENCH_BEDROCK_MODELS + AWS 자격증명 또는 OPENAI_API_KEY 설정 후 재실행)\");\n\t\t}\n\t\tSystem.out.println();\n\t\tSystem.out.printf(\"%-12s %-10s %8s %8s %8s %10s%n\", \"전략\", \"구간\", \"정밀도\", \"재현율\", \"F1\", \"LLM호출\");\n\n\t\tMap<String, Tally> ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()), cases, null);\n\n\t\tif (bedrockAvailable) {\n\t\t\tBedrockRuntimeClient bedrock = BedrockRuntimeClient.builder()\n\t\t\t\t\t.region(Region.of(System.getenv().getOrDefault(\"BENCH_BEDROCK_REGION\", \"ap-northeast-2\")))\n\t\t\t\t\t.build();\n\t\t\tfor (String modelId : bedrockModels.split(\",\")) {\n\t\t\t\trunLlmStrategies(\"bedrock:\" + modelId.strip(),\n\t\t\t\t\t\tnew BedrockLlmAdapter(bedrock, modelId.strip()), cases);\n\t\t\t}\n\t\t}\n\t\tif (openAiAvailable) {\n\t\t\trunLlmStrategies(\"openai:\" + System.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\tnew OpenAiLlmAdapter(openAiKey,\n\t\t\t\t\t\t\tSystem.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\t\t\t\"https://api.openai.com\"),\n\t\t\t\t\tcases);\n\t\t}\n\n\t\t// 룰은 결정적 — 정형 문장 전부 추출(재현율 1.0), 노이즈에서 오탐 0이어야 한다\n\t\tassertEquals(1.0, ruleResult.get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\t/** 모델 하나에 대해 llm(단독)·rule-first(폴백) 두 전략을 측정한다. */\n\tprivate void runLlmStrategies(String modelLabel, LlmPort llm, List<BenchCase> cases) {\n\t\tSystem.out.println(\"### 모델: \" + modelLabel);\n\t\tSimpleMeterRegistry llmMeters = new SimpleMeterRegistry();\n\t\trun(\"llm\", new LlmClaimExtractor(llm, objectMapper, llmMeters), cases, llmMeters);\n\n\t\tSimpleMeterRegistry hybridMeters = new SimpleMeterRegistry();\n\t\trun(\"rule-first\", new RuleFirstClaimExtractor(new RuleClaimExtractor(),\n\t\t\t\tnew LlmClaimExtractor(llm, objectMapper, hybridMeters)), cases, hybridMeters);\n\t}\n\n\tprivate Map<String, Tally> run(String name, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters) {\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\t// 파이프라인과 동일하게 룰필터를 먼저 통과시킨다\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence())) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tbyCategory.merge(benchCase.category(), new Tally((int) tp, fp, fn),\n\t\t\t\t\t(a, b) -> a.plus(b.tp(), b.fp(), b.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tdouble llmCalls = meters == null ? 0\n\t\t\t\t: meters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count();\n\t\tfor (Map.Entry<String, Tally> entry : byCategory.entrySet()) {\n\t\t\tprintRow(name, entry.getKey(), entry.getValue(), null);\n\t\t}\n\t\tprintRow(name, \"전체\", overall, llmCalls);\n\t\tSystem.out.println();\n\t\treturn byCategory;\n\t}\n\n\tprivate void printRow(String strategy, String segment, Tally tally, Double llmCalls) {\n\t\tSystem.out.printf(\"%-12s %-10s %8.2f %8.2f %8.2f %10s%n\", strategy, segment,\n\t\t\t\ttally.precision(), tally.recall(), tally.f1(), llmCalls == null ? \"-\" : \"%.0f\".formatted(llmCalls));\n\t}\n\n\t/** 채점 키 — 주어·속성·값·극성(시점·범위는 채점 제외). 공백 정규화. */\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> norm = s -> s == null ? \"\" : s.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", norm.apply(claim.subject()), norm.apply(claim.attribute()),\n\t\t\t\tnorm.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode c = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", c.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"attribute\").asString(\"\"), c.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n}\n"}, "ef4d29e08478529fed5f179aaf5f127e7f861dd4": {"t": 1785755775.2882648, "n": 6333, "h": "0b7dd3cac0d32720", "c": "package com.baton.bench;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\nimport com.baton.adapter.out.llm.LlmClaimExtractor;\nimport com.baton.adapter.out.llm.OpenAiLlmAdapter;\nimport com.baton.adapter.out.llm.RuleFirstClaimExtractor;\nimport com.baton.adapter.out.llm.RuleOnlyClaimExtractor;\nimport com.baton.application.port.out.ExtractClaimsPort;\nimport com.baton.domain.claim.BusinessSignalFilter;\nimport com.baton.domain.claim.ExtractedClaim;\nimport com.baton.domain.claim.RuleClaimExtractor;\nimport io.micrometer.core.instrument.simple.SimpleMeterRegistry;\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.junit.jupiter.api.Tag;\nimport org.junit.jupiter.api.Test;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\n\n/**\n * 클레임 추출 품질 벤치 — 전략(rule / llm / rule-first)별 정밀도·재현율·F1·LLM 호출 수 비교.\n * 골든 셋: claim-bench/cases.jsonl (한국어 업무 문장 + 수작업 라벨).\n * 실행: ./gradlew benchTest (평소 test 태스크에선 제외 — @Tag(\"bench\"))\n * OPENAI_API_KEY가 있으면 llm·rule-first(폴백 실동작)까지, 없으면 rule만 측정한다.\n * 기본 전략(app.claim.extractor) 확정은 이 결과로 한다.\n */\n@Tag(\"bench\")\nclass ClaimExtractionBenchTest {\n\n\trecord BenchCase(String category, String sentence, Set<String> expected) {\n\t}\n\n\trecord Tally(int tp, int fp, int fn) {\n\n\t\tTally plus(int addTp, int addFp, int addFn) {\n\t\t\treturn new Tally(tp + addTp, fp + addFp, fn + addFn);\n\t\t}\n\n\t\tdouble precision() {\n\t\t\treturn tp + fp == 0 ? 1.0 : (double) tp / (tp + fp);\n\t\t}\n\n\t\tdouble recall() {\n\t\t\treturn tp + fn == 0 ? 1.0 : (double) tp / (tp + fn);\n\t\t}\n\n\t\tdouble f1() {\n\t\t\tdouble p = precision(), r = recall();\n\t\t\treturn p + r == 0 ? 0.0 : 2 * p * r / (p + r);\n\t\t}\n\t}\n\n\tprivate final ObjectMapper objectMapper = new ObjectMapper();\n\tprivate final BusinessSignalFilter filter = new BusinessSignalFilter();\n\n\t@Test\n\tvoid 전략별_추출_품질_비교() throws Exception {\n\t\tList<BenchCase> cases = loadCases();\n\t\tString apiKey = System.getenv(\"OPENAI_API_KEY\");\n\t\tboolean llmAvailable = apiKey != null && !apiKey.isBlank();\n\n\t\tSystem.out.println();\n\t\tSystem.out.println(\"## 클레임 추출 벤치 (\" + cases.size() + \"문장: template/freeform/noise)\");\n\t\tif (!llmAvailable) {\n\t\t\tSystem.out.println(\"OPENAI_API_KEY 없음 — rule 전략만 측정 (llm·rule-first는 키 확보 후 재실행)\");\n\t\t}\n\t\tSystem.out.println();\n\t\tSystem.out.printf(\"%-12s %-10s %8s %8s %8s %10s%n\", \"전략\", \"구간\", \"정밀도\", \"재현율\", \"F1\", \"LLM호출\");\n\n\t\tMap<String, Tally> ruleResult = run(\"rule\", new RuleOnlyClaimExtractor(new RuleClaimExtractor()), cases, null);\n\n\t\tif (llmAvailable) {\n\t\t\tSimpleMeterRegistry llmMeters = new SimpleMeterRegistry();\n\t\t\trun(\"llm\", new LlmClaimExtractor(openAi(apiKey), objectMapper, llmMeters), cases, llmMeters);\n\n\t\t\tSimpleMeterRegistry hybridMeters = new SimpleMeterRegistry();\n\t\t\trun(\"rule-first\", new RuleFirstClaimExtractor(new RuleClaimExtractor(),\n\t\t\t\t\tnew LlmClaimExtractor(openAi(apiKey), objectMapper, hybridMeters)), cases, hybridMeters);\n\t\t}\n\n\t\t// 룰은 결정적 — 정형 문장 전부 추출(재현율 1.0), 노이즈에서 오탐 0이어야 한다\n\t\tassertEquals(1.0, ruleResult.get(\"template\").recall(), 1e-9, \"룰이 정형 문장을 놓침\");\n\t\tassertEquals(0, ruleResult.get(\"noise\").fp(), \"룰이 노이즈에서 오탐\");\n\t}\n\n\tprivate OpenAiLlmAdapter openAi(String apiKey) {\n\t\treturn new OpenAiLlmAdapter(apiKey, System.getenv().getOrDefault(\"BENCH_LLM_MODEL\", \"gpt-4o-mini\"),\n\t\t\t\t\"https://api.openai.com\");\n\t}\n\n\tprivate Map<String, Tally> run(String name, ExtractClaimsPort extractor, List<BenchCase> cases,\n\t\t\tSimpleMeterRegistry meters) {\n\t\tMap<String, Tally> byCategory = new LinkedHashMap<>();\n\t\tTally overall = new Tally(0, 0, 0);\n\t\tfor (BenchCase benchCase : cases) {\n\t\t\t// 파이프라인과 동일하게 룰필터를 먼저 통과시킨다\n\t\t\tList<ExtractedClaim> predicted = filter.pass(benchCase.sentence())\n\t\t\t\t\t? extractor.extract(List.of(benchCase.sentence())) : List.of();\n\t\t\tSet<String> predictedKeys = predicted.stream().map(this::key)\n\t\t\t\t\t.collect(Collectors.toCollection(TreeSet::new));\n\t\t\tlong tp = predictedKeys.stream().filter(benchCase.expected()::contains).count();\n\t\t\tint fp = (int) (predictedKeys.size() - tp);\n\t\t\tint fn = (int) (benchCase.expected().size() - tp);\n\t\t\tbyCategory.merge(benchCase.category(), new Tally((int) tp, fp, fn),\n\t\t\t\t\t(a, b) -> a.plus(b.tp(), b.fp(), b.fn()));\n\t\t\toverall = overall.plus((int) tp, fp, fn);\n\t\t}\n\t\tdouble llmCalls = meters == null ? 0\n\t\t\t\t: meters.counter(\"baton.llm.calls\", \"purpose\", \"claim-extract\").count();\n\t\tfor (Map.Entry<String, Tally> entry : byCategory.entrySet()) {\n\t\t\tprintRow(name, entry.getKey(), entry.getValue(), null);\n\t\t}\n\t\tprintRow(name, \"전체\", overall, llmCalls);\n\t\tSystem.out.println();\n\t\treturn byCategory;\n\t}\n\n\tprivate void printRow(String strategy, String segment, Tally tally, Double llmCalls) {\n\t\tSystem.out.printf(\"%-12s %-10s %8.2f %8.2f %8.2f %10s%n\", strategy, segment,\n\t\t\t\ttally.precision(), tally.recall(), tally.f1(), llmCalls == null ? \"-\" : \"%.0f\".formatted(llmCalls));\n\t}\n\n\t/** 채점 키 — 주어·속성·값·극성(시점·범위는 채점 제외). 공백 정규화. */\n\tprivate String key(ExtractedClaim claim) {\n\t\tFunction<String, String> norm = s -> s == null ? \"\" : s.strip().replaceAll(\"\\\\s+\", \" \");\n\t\treturn String.join(\"|\", norm.apply(claim.subject()), norm.apply(claim.attribute()),\n\t\t\t\tnorm.apply(claim.value()), claim.polarity().name());\n\t}\n\n\tprivate List<BenchCase> loadCases() throws Exception {\n\t\tList<BenchCase> cases = new ArrayList<>();\n\t\ttry (BufferedReader reader = new BufferedReader(new InputStreamReader(\n\t\t\t\tgetClass().getResourceAsStream(\"/claim-bench/cases.jsonl\"), StandardCharsets.UTF_8))) {\n\t\t\tString line;\n\t\t\twhile ((line = reader.readLine()) != null) {\n\t\t\t\tif (line.isBlank()) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tJsonNode node = objectMapper.readTree(line);\n\t\t\t\tSet<String> expected = new TreeSet<>();\n\t\t\t\tJsonNode claims = node.path(\"claims\");\n\t\t\t\tfor (int i = 0; i < claims.size(); i++) {\n\t\t\t\t\tJsonNode c = claims.get(i);\n\t\t\t\t\texpected.add(String.join(\"|\", c.path(\"subject\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"attribute\").asString(\"\"), c.path(\"value\").asString(\"\"),\n\t\t\t\t\t\t\tc.path(\"polarity\").asString(\"positive\").toUpperCase()));\n\t\t\t\t}\n\t\t\t\tcases.add(new BenchCase(node.path(\"category\").asString(\"\"),\n\t\t\t\t\t\tnode.path(\"sentence\").asString(\"\"), expected));\n\t\t\t}\n\t\t}\n\t\treturn cases;\n\t}\n}\n"}}}} |