chaos
5d215e1649
- tools/scripts/llm-key-hunter: GitHub leak hunting pipeline (hunt_*, pivot miner, two-layer verify/content caches, per-provider verification) - usable_keys: verified key vault across 12 providers (deepseek, minimax, volcanoark, longcat, codingplan, zhipu free-tier, mimo, siliconflow, etc.) - .grok/skills/llm-key-hunter: operator skill for the hunt/verify/vault flow - NewAPI channel import scripts and CDP capture helpers - Result verdict buckets (excluding multi-GB blob caches and dedup dumps)
526 lines
18 KiB
JavaScript
Executable File
526 lines
18 KiB
JavaScript
Executable File
/**
|
|
* CDP Capture — Chrome DevTools Protocol session recorder
|
|
*
|
|
* Connects to a real Chrome instance via CDP and captures:
|
|
* - All network requests (URL, method, headers, body, response)
|
|
* - All console messages
|
|
* - All user interactions (clicks, inputs, form submits)
|
|
* - All page navigations
|
|
* - Cookies & localStorage (on exit or on demand)
|
|
* - DOM snapshots & screenshots (on demand or per-navigation)
|
|
*
|
|
* Usage:
|
|
* node capture.js [options]
|
|
*
|
|
* Options:
|
|
* --port=<port> CDP port (default: 9222)
|
|
* --host=<host> CDP host (default: localhost)
|
|
* --tab=<index> Tab index to attach to (default: 0, use -1 for all tabs)
|
|
* --screenshot Take screenshot on each navigation
|
|
* --dom Save DOM snapshot on each navigation
|
|
* --bodies Capture response bodies (default: off, can be heavy)
|
|
* --filter=<regex> Only capture network requests matching this URL pattern
|
|
* --outdir=<path> Base output directory (default: ../../.. i.e. project root)
|
|
* --no-interactions Disable interaction capture
|
|
* --no-network Disable network capture
|
|
* --no-console Disable console capture
|
|
*/
|
|
|
|
const CDP = require('chrome-remote-interface');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// ─── CLI Args ───────────────────────────────────────────────
|
|
function parseArgs() {
|
|
const args = { port: 9222, host: 'localhost', tab: 0, outdir: null,
|
|
screenshot: false, dom: false, bodies: false, filter: null,
|
|
interactions: true, network: true, console: true };
|
|
|
|
for (const arg of process.argv.slice(2)) {
|
|
const [key, val] = arg.startsWith('--') ? arg.slice(2).split('=') : [arg, true];
|
|
switch (key) {
|
|
case 'port': args.port = parseInt(val); break;
|
|
case 'host': args.host = val; break;
|
|
case 'tab': args.tab = parseInt(val); break;
|
|
case 'screenshot': args.screenshot = true; break;
|
|
case 'dom': args.dom = true; break;
|
|
case 'bodies': args.bodies = true; break;
|
|
case 'filter': args.filter = val; break;
|
|
case 'outdir': args.outdir = val; break;
|
|
case 'no-interactions': args.interactions = false; break;
|
|
case 'no-network': args.network = false; break;
|
|
case 'no-console': args.console = false; break;
|
|
}
|
|
}
|
|
return args;
|
|
}
|
|
|
|
// ─── Session Setup ──────────────────────────────────────────
|
|
const args = parseArgs();
|
|
const now = new Date();
|
|
const ts = now.toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
const sessionName = `session_${ts}`;
|
|
|
|
const projectRoot = args.outdir
|
|
? path.resolve(args.outdir)
|
|
: path.resolve(__dirname, '..', '..', '..');
|
|
|
|
const dirs = {
|
|
logs: path.join(projectRoot, 'logs', sessionName),
|
|
scans: path.join(projectRoot, 'scans', 'host', sessionName),
|
|
screenshots: path.join(projectRoot, 'evidence', 'screenshots', sessionName),
|
|
loot: path.join(projectRoot, 'loot', 'credentials', sessionName),
|
|
};
|
|
|
|
for (const d of Object.values(dirs)) fs.mkdirSync(d, { recursive: true });
|
|
|
|
// ─── Output Writers ─────────────────────────────────────────
|
|
const writers = {};
|
|
function getWriter(name) {
|
|
if (!writers[name]) {
|
|
writers[name] = fs.createWriteStream(path.join(dirs.logs, `${name}.jsonl`), { flags: 'a' });
|
|
}
|
|
return writers[name];
|
|
}
|
|
|
|
function writeLine(name, obj) {
|
|
const w = getWriter(name);
|
|
w.write(JSON.stringify(obj) + '\n');
|
|
}
|
|
|
|
// ─── Interaction Injection Script ───────────────────────────
|
|
const injectScript = `
|
|
(function() {
|
|
if (window.__cdpCaptureInjected) return;
|
|
window.__cdpCaptureInjected = true;
|
|
|
|
function serialize(el) {
|
|
if (!el) return null;
|
|
return {
|
|
tag: el.tagName?.toLowerCase(),
|
|
id: el.id || undefined,
|
|
class: el.className?.toString()?.substring(0, 300) || undefined,
|
|
text: el.innerText?.substring(0, 300) || undefined,
|
|
href: el.href || undefined,
|
|
name: el.name || undefined,
|
|
type: el.type || undefined,
|
|
value: el.value?.substring(0, 1000) || undefined,
|
|
action: el.action || undefined,
|
|
method: el.method || undefined,
|
|
xpath: (function() {
|
|
try {
|
|
const s = [];
|
|
let n = el;
|
|
while (n && n.nodeType === 1) {
|
|
let idx = 1, sib = n.previousElementSibling;
|
|
while (sib) { if (sib.tagName === n.tagName) idx++; sib = sib.previousElementSibling; }
|
|
s.unshift(n.tagName.toLowerCase() + '[' + idx + ']');
|
|
n = n.parentElement;
|
|
}
|
|
return '/' + s.join('/');
|
|
} catch(e) { return undefined; }
|
|
})()
|
|
};
|
|
}
|
|
|
|
// Capture clicks
|
|
document.addEventListener('click', function(e) {
|
|
__capture(JSON.stringify({
|
|
type: 'click',
|
|
target: serialize(e.target),
|
|
timestamp: Date.now(),
|
|
url: location.href
|
|
}));
|
|
}, true);
|
|
|
|
// Capture input changes
|
|
document.addEventListener('input', function(e) {
|
|
__capture(JSON.stringify({
|
|
type: 'input',
|
|
target: serialize(e.target),
|
|
timestamp: Date.now(),
|
|
url: location.href
|
|
}));
|
|
}, true);
|
|
|
|
// Capture form submissions
|
|
document.addEventListener('submit', function(e) {
|
|
var formData = {};
|
|
try {
|
|
var fd = new FormData(e.target);
|
|
fd.forEach(function(v, k) { formData[k] = (typeof v === 'string') ? v.substring(0, 1000) : '[file]'; });
|
|
} catch(err) {}
|
|
__capture(JSON.stringify({
|
|
type: 'submit',
|
|
target: serialize(e.target),
|
|
formData: formData,
|
|
timestamp: Date.now(),
|
|
url: location.href
|
|
}));
|
|
}, true);
|
|
|
|
// Capture keydown (for Enter, etc.)
|
|
document.addEventListener('keydown', function(e) {
|
|
if (e.key === 'Enter' || e.key === 'Tab') {
|
|
__capture(JSON.stringify({
|
|
type: 'keydown',
|
|
key: e.key,
|
|
target: serialize(e.target),
|
|
timestamp: Date.now(),
|
|
url: location.href
|
|
}));
|
|
}
|
|
}, true);
|
|
})();
|
|
`;
|
|
|
|
// ─── Tab Capture ────────────────────────────────────────────
|
|
const stats = { requests: 0, responses: 0, console: 0, interactions: 0, navigations: 0, screenshots: 0, domSnapshots: 0 };
|
|
const pendingRequests = new Map();
|
|
const filterRegex = args.filter ? new RegExp(args.filter) : null;
|
|
|
|
async function attachToTarget(target, index) {
|
|
const label = `[Tab ${index}: ${target.url?.substring(0, 80)}]`;
|
|
console.log(`\n${label} Attaching...`);
|
|
|
|
const client = await CDP({ target: target.id, host: args.host, port: args.port });
|
|
const { Network, Page, Runtime, Console, DOM, Storage } = client;
|
|
|
|
// ── Enable domains ──
|
|
await Network.enable();
|
|
await Page.enable();
|
|
await Runtime.enable();
|
|
await Console.enable();
|
|
if (args.dom) await DOM.enable();
|
|
|
|
// ── Add binding for interaction capture ──
|
|
if (args.interactions) {
|
|
await Runtime.addBinding({ name: '__capture' });
|
|
}
|
|
|
|
// ── Network Events ──
|
|
if (args.network) {
|
|
Network.requestWillBeSent((params) => {
|
|
const url = params.request.url;
|
|
if (filterRegex && !filterRegex.test(url)) return;
|
|
|
|
const entry = {
|
|
requestId: params.requestId,
|
|
url: url,
|
|
method: params.request.method,
|
|
headers: params.request.headers,
|
|
postData: params.request.postData?.substring(0, 10000),
|
|
type: params.type,
|
|
initiator: params.initiator?.url,
|
|
timestamp: params.timestamp,
|
|
wallTime: params.wallTime,
|
|
};
|
|
pendingRequests.set(params.requestId, entry);
|
|
writeLine('network', { event: 'request', ...entry });
|
|
stats.requests++;
|
|
});
|
|
|
|
Network.responseReceived((params) => {
|
|
const url = params.response.url;
|
|
if (filterRegex && !filterRegex.test(url)) return;
|
|
|
|
const entry = {
|
|
requestId: params.requestId,
|
|
url: url,
|
|
status: params.response.status,
|
|
statusText: params.response.statusText,
|
|
mimeType: params.response.mimeType,
|
|
headers: params.response.headers,
|
|
remoteIP: params.response.remoteIPAddress,
|
|
remotePort: params.response.remotePort,
|
|
protocol: params.response.protocol,
|
|
timestamp: params.timestamp,
|
|
};
|
|
writeLine('network', { event: 'response', ...entry });
|
|
stats.responses++;
|
|
});
|
|
|
|
if (args.bodies) {
|
|
Network.loadingFinished(async (params) => {
|
|
try {
|
|
const { body, base64Encoded } = await Network.getResponseBody({
|
|
requestId: params.requestId,
|
|
});
|
|
const req = pendingRequests.get(params.requestId);
|
|
if (req) {
|
|
writeLine('network_bodies', {
|
|
requestId: params.requestId,
|
|
url: req.url,
|
|
body: base64Encoded ? `[base64 ${body.length} chars]` : body.substring(0, 50000),
|
|
base64Encoded,
|
|
});
|
|
}
|
|
} catch (e) { /* body not available */ }
|
|
});
|
|
}
|
|
}
|
|
|
|
// ── Console Events ──
|
|
if (args.console) {
|
|
Runtime.consoleAPICalled((params) => {
|
|
const args = params.args.map(a => a.value ?? a.description ?? a.unserializableValue ?? '');
|
|
writeLine('console', {
|
|
type: params.type,
|
|
args: args,
|
|
stackTrace: params.stackTrace?.callFrames?.[0]?.functionName,
|
|
timestamp: params.timestamp,
|
|
url: target.url,
|
|
});
|
|
stats.console++;
|
|
});
|
|
|
|
Runtime.exceptionThrown((params) => {
|
|
writeLine('console', {
|
|
type: 'exception',
|
|
text: params.exceptionDetails.text,
|
|
exception: params.exceptionDetails.exception?.description?.substring(0, 2000),
|
|
url: params.exceptionDetails.url,
|
|
lineNumber: params.exceptionDetails.lineNumber,
|
|
timestamp: Date.now() / 1000,
|
|
});
|
|
});
|
|
}
|
|
|
|
// ── Interaction Events (via binding) ──
|
|
if (args.interactions) {
|
|
Runtime.bindingCalled((params) => {
|
|
if (params.name === '__capture') {
|
|
try {
|
|
const data = JSON.parse(params.payload);
|
|
writeLine('interactions', data);
|
|
stats.interactions++;
|
|
} catch (e) { /* malformed */ }
|
|
}
|
|
});
|
|
}
|
|
|
|
// ── Navigation Events ──
|
|
Page.frameNavigated(async (params) => {
|
|
if (params.frame.parentId) return; // only top-level
|
|
const url = params.frame.url;
|
|
writeLine('navigations', {
|
|
url: url,
|
|
name: params.frame.name,
|
|
timestamp: Date.now() / 1000,
|
|
});
|
|
stats.navigations++;
|
|
console.log(`${label} Navigation → ${url.substring(0, 100)}`);
|
|
|
|
// Re-inject interaction listeners after navigation
|
|
if (args.interactions) {
|
|
try {
|
|
await Runtime.evaluate({ expression: injectScript, returnByValue: false });
|
|
} catch (e) { /* page not ready */ }
|
|
}
|
|
|
|
// Screenshot on navigation
|
|
if (args.screenshot) {
|
|
try {
|
|
await takeScreenshot(client, index, url);
|
|
} catch (e) { /* timing */ }
|
|
}
|
|
|
|
// DOM snapshot on navigation
|
|
if (args.dom) {
|
|
try {
|
|
await saveDOMSnapshot(client, index, url);
|
|
} catch (e) { /* timing */ }
|
|
}
|
|
});
|
|
|
|
// ── Inject listeners on load ──
|
|
Page.loadEventFired(async () => {
|
|
if (args.interactions) {
|
|
try {
|
|
await Runtime.evaluate({ expression: injectScript, returnByValue: false });
|
|
console.log(`${label} Interaction listeners injected.`);
|
|
} catch (e) { /* page not ready */ }
|
|
}
|
|
});
|
|
|
|
// Initial injection (in case page is already loaded)
|
|
if (args.interactions) {
|
|
try {
|
|
await Runtime.evaluate({ expression: injectScript, returnByValue: false });
|
|
} catch (e) { /* page not ready */ }
|
|
}
|
|
|
|
console.log(`${label} Capturing. (network=${args.network}, console=${args.console}, interactions=${args.interactions})`);
|
|
return client;
|
|
}
|
|
|
|
// ─── Screenshot ──────────────────────────────────────────────
|
|
async function takeScreenshot(client, tabIndex, url) {
|
|
const { Page } = client;
|
|
const { data } = await Page.captureScreenshot({ format: 'png' });
|
|
const fname = `tab${tabIndex}_${Date.now()}.png`;
|
|
fs.writeFileSync(path.join(dirs.screenshots, fname), Buffer.from(data, 'base64'));
|
|
writeLine('screenshots', { file: fname, url, timestamp: Date.now() / 1000 });
|
|
stats.screenshots++;
|
|
console.log(` 📸 Screenshot: ${fname}`);
|
|
}
|
|
|
|
// ─── DOM Snapshot ────────────────────────────────────────────
|
|
async function saveDOMSnapshot(client, tabIndex, url) {
|
|
const { DOM, Page } = client;
|
|
const { root } = await DOM.getDocument({ depth: -1 });
|
|
const html = await DOM.getOuterHTML({ nodeId: root.nodeId });
|
|
const fname = `tab${tabIndex}_${Date.now()}.html`;
|
|
fs.writeFileSync(path.join(dirs.scans, fname), html.outerHTML || String(html));
|
|
stats.domSnapshots++;
|
|
console.log(` 📄 DOM snapshot: ${fname}`);
|
|
}
|
|
|
|
// ─── Cookie & Storage Extraction ─────────────────────────────
|
|
async function extractCookies(clients) {
|
|
console.log('\nExtracting cookies & storage...');
|
|
for (let i = 0; i < clients.length; i++) {
|
|
const client = clients[i];
|
|
if (!client) continue;
|
|
try {
|
|
const { Network, Runtime, Storage } = client;
|
|
|
|
// Cookies via Network domain
|
|
const { cookies } = await Network.getCookies({});
|
|
fs.writeFileSync(
|
|
path.join(dirs.loot, `cookies_tab${i}.json`),
|
|
JSON.stringify(cookies, null, 2)
|
|
);
|
|
|
|
// localStorage
|
|
const lsResult = await Runtime.evaluate({
|
|
expression: `JSON.stringify(Object.fromEntries(Object.entries(localStorage)))`,
|
|
returnByValue: true,
|
|
});
|
|
if (lsResult.result?.value) {
|
|
fs.writeFileSync(
|
|
path.join(dirs.loot, `localStorage_tab${i}.json`),
|
|
lsResult.result.value
|
|
);
|
|
}
|
|
|
|
// sessionStorage
|
|
const ssResult = await Runtime.evaluate({
|
|
expression: `JSON.stringify(Object.fromEntries(Object.entries(sessionStorage)))`,
|
|
returnByValue: true,
|
|
});
|
|
if (ssResult.result?.value) {
|
|
fs.writeFileSync(
|
|
path.join(dirs.loot, `sessionStorage_tab${i}.json`),
|
|
ssResult.result.value
|
|
);
|
|
}
|
|
|
|
console.log(` Tab ${i}: ${cookies.length} cookies, localStorage + sessionStorage saved`);
|
|
} catch (e) {
|
|
console.log(` Tab ${i}: extraction failed (${e.message})`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Main ────────────────────────────────────────────────────
|
|
async function main() {
|
|
console.log('╔══════════════════════════════════════════════════╗');
|
|
console.log('║ CDP Capture — Session Recorder ║');
|
|
console.log('╚══════════════════════════════════════════════════╝');
|
|
console.log(`\nSession: ${sessionName}`);
|
|
console.log(`Output: ${projectRoot}`);
|
|
console.log(`CDP: ${args.host}:${args.port}`);
|
|
|
|
// Get browser targets
|
|
let targets;
|
|
try {
|
|
targets = await CDP.List({ host: args.host, port: args.port });
|
|
} catch (e) {
|
|
console.error(`\n❌ Cannot connect to CDP at ${args.host}:${args.port}`);
|
|
console.error(` Make sure Chrome is running with --remote-debugging-port=${args.port}`);
|
|
console.error(` Use: tools/scripts/launch-chrome.ps1`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const pageTargets = targets.filter(t => t.type === 'page');
|
|
if (pageTargets.length === 0) {
|
|
console.error('\n❌ No browser tabs found. Open a tab in Chrome first.');
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(`\nFound ${pageTargets.length} tab(s):`);
|
|
pageTargets.forEach((t, i) => {
|
|
console.log(` [${i}] ${t.url?.substring(0, 100)}`);
|
|
});
|
|
|
|
// Select tabs to attach
|
|
let tabsToAttach;
|
|
if (args.tab === -1) {
|
|
tabsToAttach = pageTargets.map((t, i) => ({ target: t, index: i }));
|
|
} else if (args.tab >= 0 && args.tab < pageTargets.length) {
|
|
tabsToAttach = [{ target: pageTargets[args.tab], index: args.tab }];
|
|
} else {
|
|
console.error(`\n❌ Invalid tab index: ${args.tab}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
// Attach to targets
|
|
const clients = [];
|
|
for (const { target, index } of tabsToAttach) {
|
|
try {
|
|
const client = await attachToTarget(target, index);
|
|
clients.push(client);
|
|
} catch (e) {
|
|
console.error(`Tab ${index}: attach failed — ${e.message}`);
|
|
clients.push(null);
|
|
}
|
|
}
|
|
|
|
console.log(`\n✅ Capturing ${clients.filter(Boolean).length} tab(s). Press Ctrl+C to stop.\n`);
|
|
|
|
// ── Graceful Shutdown ──
|
|
let shuttingDown = false;
|
|
process.on('SIGINT', async () => {
|
|
if (shuttingDown) return;
|
|
shuttingDown = true;
|
|
console.log('\n\nShutting down...');
|
|
|
|
// Extract cookies & storage
|
|
await extractCookies(clients);
|
|
|
|
// Write summary
|
|
const summary = {
|
|
session: sessionName,
|
|
startedAt: now.toISOString(),
|
|
endedAt: new Date().toISOString(),
|
|
args: args,
|
|
stats: stats,
|
|
outputDirs: dirs,
|
|
};
|
|
fs.writeFileSync(path.join(dirs.logs, 'summary.json'), JSON.stringify(summary, null, 2));
|
|
console.log(`\n📊 Session Summary:`);
|
|
console.log(` Network requests: ${stats.requests}`);
|
|
console.log(` Network responses: ${stats.responses}`);
|
|
console.log(` Console messages: ${stats.console}`);
|
|
console.log(` Interactions: ${stats.interactions}`);
|
|
console.log(` Navigations: ${stats.navigations}`);
|
|
console.log(` Screenshots: ${stats.screenshots}`);
|
|
console.log(` DOM snapshots: ${stats.domSnapshots}`);
|
|
console.log(`\n📁 Output: ${dirs.logs}`);
|
|
|
|
// Close writers
|
|
for (const w of Object.values(writers)) w.end();
|
|
|
|
// Close clients
|
|
for (const c of clients) { if (c) try { await c.close(); } catch (e) {} }
|
|
|
|
process.exit(0);
|
|
});
|
|
}
|
|
|
|
main().catch(e => {
|
|
console.error('Fatal error:', e);
|
|
process.exit(1);
|
|
});
|