OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
20 KiB
JSON
1 line
20 KiB
JSON
{"repo": "anslationdev/altftool", "files": {"altftoolweb/src/tools/llm-cost-privacy-analyzer/lib/analyzeUsage.mjs": {"a269da1fe0f96d32cc842b25cc065991bb8fe481": {"t": 1786068452.6083689, "n": 14948, "h": "a5a2dc0ca9606286", "c": "import {\n DEFAULT_ENABLED_TYPES,\n redactText,\n} from \"../../universal-pii-ai-redactor/lib/redact.mjs\";\n\nexport const SAMPLE_RATE_PRESETS = Object.freeze({\n openai: {\n key: \"openai\",\n label: \"OpenAI Models (GPT-4o, GPT-4o-mini, o1)\",\n rates: {\n \"gpt-4o\": { inputPerMillion: 2.5, outputPerMillion: 10.0 },\n \"gpt-4o-mini\": { inputPerMillion: 0.15, outputPerMillion: 0.6 },\n \"o1-preview\": { inputPerMillion: 15.0, outputPerMillion: 60.0 },\n \"o1-mini\": { inputPerMillion: 3.0, outputPerMillion: 12.0 },\n \"*\": { inputPerMillion: 2.5, outputPerMillion: 10.0 },\n },\n },\n anthropic: {\n key: \"anthropic\",\n label: \"Anthropic Models (Claude 3.5 Sonnet, Haiku, Opus)\",\n rates: {\n \"claude-3-5-sonnet\": { inputPerMillion: 3.0, outputPerMillion: 15.0 },\n \"claude-3-haiku\": { inputPerMillion: 0.25, outputPerMillion: 1.25 },\n \"claude-3-opus\": { inputPerMillion: 15.0, outputPerMillion: 75.0 },\n \"*\": { inputPerMillion: 3.0, outputPerMillion: 15.0 },\n },\n },\n google: {\n key: \"google\",\n label: \"Google Gemini (1.5 Pro, 1.5 Flash)\",\n rates: {\n \"gemini-1.5-pro\": { inputPerMillion: 1.25, outputPerMillion: 5.0 },\n \"gemini-1.5-flash\": { inputPerMillion: 0.075, outputPerMillion: 0.3 },\n \"*\": { inputPerMillion: 1.25, outputPerMillion: 5.0 },\n },\n },\n openSource: {\n key: \"openSource\",\n label: \"DeepSeek & Meta Llama 3\",\n rates: {\n \"deepseek-chat\": { inputPerMillion: 0.14, outputPerMillion: 0.28 },\n \"deepseek-reasoner\": { inputPerMillion: 0.55, outputPerMillion: 2.19 },\n \"llama-3-70b\": { inputPerMillion: 0.59, outputPerMillion: 0.79 },\n \"*\": { inputPerMillion: 0.2, outputPerMillion: 0.5 },\n },\n },\n});\n\nexport const SECRET_PATTERNS = Object.freeze([\n {\n type: \"openai_key\",\n label: \"OpenAI API Key\",\n regex: /sk-[a-zA-Z0-9]{32,64}/g,\n severity: \"high\",\n },\n {\n type: \"anthropic_key\",\n label: \"Anthropic API Key\",\n regex: /sk-ant-[a-zA-Z0-9\\-_]{32,64}/g,\n severity: \"high\",\n },\n {\n type: \"gemini_key\",\n label: \"Google Gemini API Key\",\n regex: /AIzaSy[a-zA-Z0-9\\-_]{33}/g,\n severity: \"high\",\n },\n {\n type: \"github_token\",\n label: \"GitHub Personal Access Token\",\n regex: /ghp_[a-zA-Z0-9]{36}/g,\n severity: \"high\",\n },\n {\n type: \"aws_key\",\n label: \"AWS Access Key\",\n regex: /\\bAKIA[0-9A-Z]{16}\\b/g,\n severity: \"high\",\n },\n {\n type: \"jwt_token\",\n label: \"JWT Bearer Token\",\n regex: /\\beyJ[a-zA-Z0-9\\-_]+\\.[a-zA-Z0-9\\-_]+\\.[a-zA-Z0-9\\-_]+\\b/g,\n severity: \"high\",\n },\n]);\n\nconst MAX_SOURCE_CHARACTERS = 8 * 1024 * 1024;\nconst MAX_RECORDS = 5_000;\nconst MAX_PRIVACY_CHARACTERS = 500_000;\n\nconst MODEL_PATHS = [\"model\", \"model_name\", \"modelName\", \"request.model\", \"response.model\"];\nconst INPUT_TOKEN_PATHS = [\n \"usage.input_tokens\",\n \"usage.inputTokens\",\n \"usage.prompt_tokens\",\n \"usage.promptTokens\",\n \"input_tokens\",\n \"inputTokens\",\n \"prompt_tokens\",\n \"promptTokens\",\n];\nconst OUTPUT_TOKEN_PATHS = [\n \"usage.output_tokens\",\n \"usage.outputTokens\",\n \"usage.completion_tokens\",\n \"usage.completionTokens\",\n \"output_tokens\",\n \"outputTokens\",\n \"completion_tokens\",\n \"completionTokens\",\n];\nconst TOTAL_TOKEN_PATHS = [\n \"usage.total_tokens\",\n \"usage.totalTokens\",\n \"total_tokens\",\n \"totalTokens\",\n];\nconst CONTENT_KEYS = new Set([\n \"content\",\n \"input\",\n \"inputs\",\n \"message\",\n \"messages\",\n \"prompt\",\n \"prompts\",\n \"requestbody\",\n \"responsebody\",\n \"text\",\n]);\n\nfunction isObject(value) {\n return Boolean(value) && typeof value === \"object\" && !Array.isArray(value);\n}\n\nfunction getPath(value, path) {\n return path.split(\".\").reduce((current, key) => current?.[key], value);\n}\n\nfunction firstValue(value, paths) {\n for (const path of paths) {\n const candidate = getPath(value, path);\n if (candidate !== undefined && candidate !== null && candidate !== \"\") return candidate;\n }\n return undefined;\n}\n\nfunction tokenNumber(value) {\n const number = Number(value);\n return Number.isFinite(number) && number >= 0 ? Math.round(number) : 0;\n}\n\nfunction normalizeKey(value) {\n return String(value || \"\")\n .replace(/[^a-zA-Z0-9]/g, \"\")\n .toLowerCase();\n}\n\nfunction parseDelimitedLine(line, delimiter) {\n const cells = [];\n let cell = \"\";\n let quoted = false;\n for (let index = 0; index < line.length; index += 1) {\n const character = line[index];\n if (quoted) {\n if (character === '\"' && line[index + 1] === '\"') {\n cell += '\"';\n index += 1;\n } else if (character === '\"') {\n quoted = false;\n } else {\n cell += character;\n }\n } else if (character === '\"') {\n quoted = true;\n } else if (character === delimiter) {\n cells.push(cell);\n cell = \"\";\n } else {\n cell += character;\n }\n }\n cells.push(cell);\n return cells;\n}\n\nfunction parseCsv(text) {\n const lines = text.split(/\\r?\\n/).filter((line) => line.trim());\n if (lines.length < 2) return [];\n const delimiter = (lines[0].match(/\\t/g) || []).length > (lines[0].match(/,/g) || []).length\n ? \"\\t\"\n : \",\";\n const headers = parseDelimitedLine(lines[0], delimiter).map((header) => header.trim());\n return lines.slice(1).map((line) =>\n Object.fromEntries(\n headers.map((header, index) => [\n header,\n parseDelimitedLine(line, delimiter)[index]?.trim() || \"\",\n ]),\n ),\n );\n}\n\nfunction extractArray(value) {\n if (Array.isArray(value)) return value;\n if (!isObject(value)) return null;\n for (const key of [\"data\", \"events\", \"items\", \"logs\", \"records\", \"requests\", \"usage\"]) {\n if (Array.isArray(value[key])) return value[key];\n }\n return [value];\n}\n\nexport function parseUsageLogs(source) {\n const raw = String(source || \"\");\n const bounded = raw.slice(0, MAX_SOURCE_CHARACTERS);\n const trimmed = bounded.trim();\n if (!trimmed) return { ok: false, error: \"Open or paste an AI usage log first.\" };\n\n let records = [];\n let format = \"json\";\n if (/^[{[]/.test(trimmed)) {\n try {\n records = extractArray(JSON.parse(trimmed)) || [];\n } catch {\n const lines = trimmed.split(/\\r?\\n/).filter(Boolean);\n try {\n records = lines.map((line) => JSON.parse(line));\n format = \"jsonl\";\n } catch {\n return { ok: false, error: \"The log is not valid JSON or JSONL.\" };\n }\n }\n } else {\n records = parseCsv(trimmed);\n format = \"csv\";\n }\n\n const objects = records.filter(isObject).slice(0, MAX_RECORDS);\n if (!objects.length) {\n return { ok: false, error: \"No object-shaped usage records were found.\" };\n }\n return {\n ok: true,\n records: objects,\n format,\n truncated: raw.length > bounded.length || records.length > MAX_RECORDS,\n };\n}\n\nfunction flattenRecord(record) {\n const flat = {};\n function walk(value, path = \"\", depth = 0) {\n if (depth > 5 || !isObject(value)) return;\n for (const [key, item] of Object.entries(value)) {\n const nextPath = path ? `${path}.${key}` : key;\n if (isObject(item)) walk(item, nextPath, depth + 1);\n else flat[nextPath] = item;\n }\n }\n walk(record);\n return flat;\n}\n\nfunction recordField(record, paths) {\n const direct = firstValue(record, paths);\n if (direct !== undefined) return direct;\n const flat = flattenRecord(record);\n const normalized = new Map(\n Object.entries(flat).map(([key, value]) => [normalizeKey(key), value]),\n );\n for (const path of paths) {\n const found = normalized.get(normalizeKey(path));\n if (found !== undefined) return found;\n }\n return undefined;\n}\n\nfunction collectContentStrings(value, state, key = \"\", depth = 0) {\n if (state.characters >= MAX_PRIVACY_CHARACTERS || depth > 8) return;\n if (typeof value === \"string\") {\n if (CONTENT_KEYS.has(normalizeKey(key))) {\n const remaining = MAX_PRIVACY_CHARACTERS - state.characters;\n const bounded = value.slice(0, remaining);\n state.values.push(bounded);\n state.characters += bounded.length;\n if (bounded.length < value.length) state.truncated = true;\n }\n return;\n }\n if (Array.isArray(value)) {\n value.forEach((item) => collectContentStrings(item, state, key, depth + 1));\n return;\n }\n if (!isObject(value)) return;\n Object.entries(value).forEach(([childKey, item]) =>\n collectContentStrings(item, state, childKey, depth + 1),\n );\n}\n\nexport function parseRateTable(source) {\n const text = String(source || \"\").trim();\n if (!text) return { ok: true, rates: new Map(), warnings: [\"No token rates supplied.\"] };\n let parsed;\n try {\n parsed = JSON.parse(text);\n } catch {\n return { ok: false, error: \"The token-rate table is not valid JSON.\" };\n }\n\n const rows = Array.isArray(parsed)\n ? parsed\n : isObject(parsed)\n ? Object.entries(parsed).map(([model, value]) => ({ model, ...value }))\n : [];\n const rates = new Map();\n const errors = [];\n for (const [index, row] of rows.entries()) {\n const model = String(row?.model ?? row?.modelPattern ?? \"\").trim();\n const inputPerMillion = Number(\n row?.inputPerMillion ?? row?.input_per_million ?? row?.input,\n );\n const outputPerMillion = Number(\n row?.outputPerMillion ?? row?.output_per_million ?? row?.output,\n );\n if (\n !model ||\n !Number.isFinite(inputPerMillion) ||\n inputPerMillion < 0 ||\n !Number.isFinite(outputPerMillion) ||\n outputPerMillion < 0\n ) {\n errors.push(\n `Rate row ${index + 1} needs model, inputPerMillion, and outputPerMillion as non-negative numbers.`,\n );\n continue;\n }\n rates.set(model.toLowerCase(), { model, inputPerMillion, outputPerMillion });\n }\n return errors.length ? { ok: false, error: errors.join(\" \") } : { ok: true, rates, warnings: [] };\n}\n\nfunction rateFor(model, rates) {\n return rates.get(model.toLowerCase()) || rates.get(\"*\") || null;\n}\n\nexport function analyzeUsageLogs(logSource, rateSource = \"\") {\n const parsed = parseUsageLogs(logSource);\n if (!parsed.ok) return parsed;\n const parsedRates = parseRateTable(rateSource);\n if (!parsedRates.ok) return parsedRates;\n\n const privacyState = { values: [], characters: 0, truncated: false };\n const records = parsed.records.map((record, index) => {\n const model = String(recordField(record, MODEL_PATHS) || \"Unknown model\").slice(0, 200);\n const inputTokens = tokenNumber(recordField(record, INPUT_TOKEN_PATHS));\n const outputTokens = tokenNumber(recordField(record, OUTPUT_TOKEN_PATHS));\n const declaredTotal = tokenNumber(recordField(record, TOTAL_TOKEN_PATHS));\n const totalTokens = Math.max(declaredTotal, inputTokens + outputTokens);\n const unallocatedTokens = Math.max(0, totalTokens - inputTokens - outputTokens);\n const rate = rateFor(model, parsedRates.rates);\n const priceable = Boolean(rate) && unallocatedTokens === 0;\n const estimatedCost = priceable\n ? (inputTokens / 1_000_000) * rate.inputPerMillion +\n (outputTokens / 1_000_000) * rate.outputPerMillion\n : null;\n collectContentStrings(record, privacyState);\n return {\n index: index + 1,\n model,\n inputTokens,\n outputTokens,\n totalTokens,\n unallocatedTokens,\n estimatedCost,\n rateMatched: Boolean(rate),\n priceable,\n };\n });\n\n const privacyTotals = new Map();\n for (const text of privacyState.values) {\n const scan = redactText(text, {\n enabledTypes: DEFAULT_ENABLED_TYPES,\n mode: \"label\",\n });\n for (const item of scan.summary) {\n const current = privacyTotals.get(item.type) || {\n type: item.type,\n label: item.label,\n count: 0,\n };\n current.count += item.count;\n privacyTotals.set(item.type, current);\n }\n }\n\n const modelMap = new Map();\n for (const record of records) {\n const current = modelMap.get(record.model) || {\n model: record.model,\n requestCount: 0,\n inputTokens: 0,\n outputTokens: 0,\n totalTokens: 0,\n unallocatedTokens: 0,\n estimatedCost: 0,\n pricedRequestCount: 0,\n };\n current.requestCount += 1;\n current.inputTokens += record.inputTokens;\n current.outputTokens += record.outputTokens;\n current.totalTokens += record.totalTokens;\n current.unallocatedTokens += record.unallocatedTokens;\n current.estimatedCost += record.estimatedCost || 0;\n current.pricedRequestCount += Number(record.priceable);\n modelMap.set(record.model, current);\n }\n\n const models = [...modelMap.values()].sort(\n (left, right) => right.totalTokens - left.totalTokens || left.model.localeCompare(right.model),\n );\n return {\n ok: true,\n format: parsed.format,\n records,\n models,\n requestCount: records.length,\n modelCount: models.length,\n inputTokens: records.reduce((sum, record) => sum + record.inputTokens, 0),\n outputTokens: records.reduce((sum, record) => sum + record.outputTokens, 0),\n totalTokens: records.reduce((sum, record) => sum + record.totalTokens, 0),\n unallocatedTokens: records.reduce((sum, record) => sum + record.unallocatedTokens, 0),\n estimatedCost: records.reduce((sum, record) => sum + (record.estimatedCost || 0), 0),\n pricedRequestCount: records.filter((record) => record.priceable).length,\n unmatchedRateRequestCount: records.filter((record) => !record.rateMatched).length,\n privacySignals: [...privacyTotals.values()].sort((left, right) => right.count - left.count),\n privacySignalCount: [...privacyTotals.values()].reduce(\n (sum, item) => sum + item.count,\n 0,\n ),\n scannedContentCharacters: privacyState.characters,\n truncated: parsed.truncated || privacyState.truncated,\n rateWarningCount: parsedRates.warnings.length,\n };\n}\n\nexport function buildUsageReport(result) {\n if (!result?.ok) return null;\n return {\n generatedAt: new Date().toISOString(),\n format: result.format,\n requestCount: result.requestCount,\n modelCount: result.modelCount,\n inputTokens: result.inputTokens,\n outputTokens: result.outputTokens,\n totalTokens: result.totalTokens,\n unallocatedTokens: result.unallocatedTokens,\n estimatedCost: result.estimatedCost,\n pricedRequestCount: result.pricedRequestCount,\n unmatchedRateRequestCount: result.unmatchedRateRequestCount,\n privacySignalCount: result.privacySignalCount,\n privacySignals: result.privacySignals,\n modelSummaries: result.models.map((model, index) => ({\n model: index + 1,\n requestCount: model.requestCount,\n inputTokens: model.inputTokens,\n outputTokens: model.outputTokens,\n totalTokens: model.totalTokens,\n unallocatedTokens: model.unallocatedTokens,\n estimatedCost: model.estimatedCost,\n pricedRequestCount: model.pricedRequestCount,\n })),\n truncated: result.truncated,\n note:\n \"Costs use only user-supplied rates and recorded token counts. Privacy signals are pattern matches, not proof that all sensitive data was found. This report excludes model names and log content.\",\n };\n}\n"}}, ".github/workflows/deployment-readiness.yml": {"a269da1fe0f96d32cc842b25cc065991bb8fe481": {"t": 1786068537.5811937, "n": 4625, "h": "ec34d62ac569c103", "c": "name: Deployment Readiness\n\non:\n workflow_dispatch:\n\nenv:\n CI: true\n FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true\n NEXT_TELEMETRY_DISABLED: 1\n ALTFT_ENV_STRICT: true\n ALTFT_ENV_REQUIRE_MONITORING: true\n\njobs:\n environment:\n runs-on: ubuntu-latest\n timeout-minutes: 25\n\n env:\n NEXT_PUBLIC_FIREBASE_API_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_API_KEY || vars.NEXT_PUBLIC_FIREBASE_API_KEY }}\n NEXT_PUBLIC_FIREBASE_PROJECT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_PROJECT_ID || vars.NEXT_PUBLIC_FIREBASE_PROJECT_ID }}\n NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET: ${{ secrets.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET || vars.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET }}\n NEXT_PUBLIC_FIREBASE_APP_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_APP_ID || vars.NEXT_PUBLIC_FIREBASE_APP_ID }}\n NEXT_PUBLIC_FIREBASE_VAPID_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_VAPID_KEY || vars.NEXT_PUBLIC_FIREBASE_VAPID_KEY }}\n NEXT_PUBLIC_GOOGLE_MAPS_API_KEY: ${{ secrets.NEXT_PUBLIC_GOOGLE_MAPS_API_KEY || vars.NEXT_PUBLIC_GOOGLE_MAPS_API_KEY }}\n FIREBASE_PROJECT_ID: ${{ secrets.FIREBASE_PROJECT_ID || vars.FIREBASE_PROJECT_ID }}\n FIREBASE_CLIENT_EMAIL: ${{ secrets.FIREBASE_CLIENT_EMAIL }}\n FIREBASE_PRIVATE_KEY: ${{ secrets.FIREBASE_PRIVATE_KEY }}\n GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}\n OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}\n NEXT_PUBLIC_DEEPSEEK_API_KEY: ${{ secrets.NEXT_PUBLIC_DEEPSEEK_API_KEY }}\n ALTFT_MONITOR_WEB_URL: ${{ vars.ALTFT_MONITOR_WEB_URL || 'https://www.altftool.com' }}\n ALTFT_MONITOR_ADMIN_URL: ${{ vars.ALTFT_MONITOR_ADMIN_URL || 'https://www.tier2.anslation.com' }}\n ALTFT_MONITOR_ADMIN_TOKEN: ${{ secrets.ALTFT_MONITOR_ADMIN_TOKEN }}\n ALTFT_MONITOR_ADMIN_EMAIL: ${{ secrets.ALTFT_MONITOR_ADMIN_EMAIL }}\n ALTFT_MONITOR_ADMIN_PASSWORD: ${{ secrets.ALTFT_MONITOR_ADMIN_PASSWORD }}\n ALTFT_MONITOR_FIREBASE_API_KEY: ${{ secrets.ALTFT_MONITOR_FIREBASE_API_KEY || secrets.NEXT_PUBLIC_FIREBASE_API_KEY }}\n ALTFT_MONITOR_ALERT_WEBHOOK_URL: ${{ secrets.ALTFT_MONITOR_ALERT_WEBHOOK_URL }}\n VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}\n VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}\n VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}\n VERCEL_WEB_PROJECT_ID: ${{ secrets.VERCEL_WEB_PROJECT_ID }}\n VERCEL_ADMIN_PROJECT_ID: ${{ secrets.VERCEL_ADMIN_PROJECT_ID }}\n\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Setup Node\n uses: actions/setup-node@v4\n with:\n node-version-file: .nvmrc\n cache: npm\n cache-dependency-path: package-lock.json\n\n - name: Install dependencies\n run: npm ci\n\n - name: Verify deployment environment\n run: npm run env:readiness:strict\n\n - name: Verify legacy Vercel deploy secrets\n if: vars.ALTFT_ENABLE_VERCEL_ADMIN_DEPLOY == 'true'\n env:\n VERCEL_READINESS_OUTPUT_PATH: vercel-deploy-readiness-report.json\n run: npm run deploy:readiness -- --target=all\n\n - name: Build web and admin for performance budgets\n run: npm run build\n\n - name: Generate performance budget report\n run: npm run performance:budget:strict -- --output performance-budget-report.json --output-md performance-budget-report.md\n\n - name: Verify Firebase data integrity\n run: npm run firebase:integrity:strict -- --output firebase-integrity-report.json --output-md firebase-integrity-report.md\n\n - name: Verify production links and images\n env:\n ALTFT_LINK_CHECK_URL: ${{ vars.ALTFT_MONITOR_WEB_URL || 'https://www.altftool.com' }}\n run: npm run monitor:links:strict -- --limit 12 --output production-links-report.json --output-md production-links-report.md\n\n - name: Run release doctor gate\n env:\n ALTFT_RELEASE_DOCTOR_WEB_URL: ${{ vars.ALTFT_MONITOR_WEB_URL || 'https://www.altftool.com' }}\n run: npm run release:doctor:strict -- --output release-doctor-report.json --output-md release-doctor-report.md\n\n - name: Upload deployment readiness reports\n if: always()\n uses: actions/upload-artifact@v4\n with:\n name: deployment-readiness-reports\n path: |\n vercel-deploy-readiness-report.json\n performance-budget-report.json\n performance-budget-report.md\n firebase-integrity-report.json\n firebase-integrity-report.md\n production-links-report.json\n production-links-report.md\n release-doctor-report.json\n release-doctor-report.md\n if-no-files-found: ignore\n"}}}} |