1 line
16 KiB
JSON
1 line
16 KiB
JSON
{"repo": "celestialbrain/leakguard", "files": {"cli.py": {"ffba09f22bc12a8bfc04ad05ac42a0636351990f": {"t": 1785674460.611741, "n": 15011, "h": "05f1158c617e73a7", "c": "\"\"\"\nleakguard CLI — subcommands:\n\n scan run sources, populate the store\n list show findings (tokens redacted by default)\n show reveal full token for one finding\n mark update status (reported / revoked / false_positive / ignored)\n owners group new findings by owner — the email worklist\n\nscan defaults to --dry-run; pass --send to actually email providers.\n\"\"\"\nfrom __future__ import annotations\n\nimport argparse\nimport os\nimport sys\nfrom pathlib import Path\n\nfrom .discovery import DiscoverySession\nfrom .pipeline import Pipeline\nfrom .reporters import EmailReporter, StdoutReporter\nfrom .sources.github import GitHubSource\nfrom .sources.github_events import GitHubEventsSource\nfrom .sources.gitlab import GitLabSource\nfrom .sources.google import GoogleSource\nfrom .store import DEFAULT_DB_PATH, FindingStore, VALID_STATUSES\n\n\nDEFAULT_GITHUB_QUERIES = (\n # AI / LLM\n '\"sk-proj-\" extension:env',\n '\"sk-svcacct-\"',\n '\"sk-ant-api03-\"',\n '\"hf_\" extension:env',\n '\"r8_\" extension:env',\n # Cloud infra\n '\"AKIA\" path:.env',\n '\"ASIA\" path:.env',\n '\"dop_v1_\"',\n '\"AIza\" extension:env',\n '\"hvs.\"',\n '\"atlasv1\"',\n '\"dapi\" extension:env',\n # Dev platforms\n '\"ghp_\" extension:env',\n '\"github_pat_\" extension:env',\n '\"glpat-\" extension:env',\n '\"npm_\" extension:env',\n '\"sntrys_\"',\n # Comms / SaaS\n '\"xoxb-\" extension:env',\n '\"xoxp-\" extension:env',\n '\"SG.\" extension:env',\n '\"shpat_\"',\n '\"lin_api_\"',\n # Financial\n '\"sk_live_\" extension:env',\n '\"rk_live_\"',\n '\"sq0atp-\"',\n '\"sq0csp-\"',\n)\n\nDEFAULT_GITLAB_QUERIES = (\n # Scoped search has no path/extension filters; rely on distinctive prefixes.\n 'sk-proj-',\n 'sk-ant-api03-',\n 'sk-svcacct-',\n 'AKIA',\n 'dop_v1_',\n 'ghp_',\n 'github_pat_',\n 'glpat-',\n 'hvs.',\n 'sntrys_',\n 'sk_live_',\n 'shpat_',\n 'lin_api_',\n)\n\nDEFAULT_GOOGLE_QUERIES = (\n # GitHub surfaces we miss via the Code Search path (cached pages, forks).\n 'site:github.com \"sk-proj-\"',\n 'site:gist.github.com \"sk-ant-api03-\"',\n # Non-GitHub git hosts.\n 'site:gitlab.com \"sk-proj-\"',\n 'site:gitlab.com \"AKIA\"',\n 'site:bitbucket.org \"sk-proj-\"',\n 'site:codeberg.org \"OPENAI_API_KEY=\"',\n # Paste sites.\n 'site:pastebin.com \"sk-proj-\"',\n 'site:pastebin.com \"AKIA\"',\n 'site:hastebin.com \"sk-ant-api03-\"',\n # Everything else Google indexes.\n '\"AKIA\" \"AWS_SECRET_ACCESS_KEY\" ext:env',\n '\"sk_live_\" ext:env',\n '\"glpat-\" ext:env',\n)\n\n\ndef main(argv: list[str] | None = None) -> int:\n parser = argparse.ArgumentParser(prog=\"leakguard\")\n parser.add_argument(\"--db\", type=Path, default=DEFAULT_DB_PATH,\n help=\"Findings DB path.\")\n sub = parser.add_subparsers(dest=\"cmd\", required=True)\n\n p_scan = sub.add_parser(\"scan\", help=\"Run scanners and populate the store.\")\n p_scan.add_argument(\"--source\",\n choices=(\"github\", \"github_events\", \"gitlab\",\n \"google\", \"all\"),\n default=\"github\")\n p_scan.add_argument(\"--query\", action=\"append\")\n p_scan.add_argument(\"--dork-file\", type=Path,\n help=\"File of queries, one per line. Use with \"\n \"leakguard/data/gitdorker_medium.txt for 240 \"\n \"curated GitHub search terms (GitDorker corpus).\")\n p_scan.add_argument(\"--limit\", type=int, default=30)\n p_scan.add_argument(\"--gitlab-group\",\n help=\"GitLab group id or full path (scopes blob search, works on free tier).\")\n p_scan.add_argument(\"--gitlab-project\",\n help=\"GitLab project id or full path (scopes blob search, works on free tier).\")\n p_scan.add_argument(\"--send\", action=\"store_true\",\n help=\"Submit to provider abuse channels. Otherwise dry-run.\")\n\n p_list = sub.add_parser(\"list\", help=\"List findings.\")\n p_list.add_argument(\"--status\", default=\"new\",\n help=f\"Filter. One of {VALID_STATUSES} or 'all'. Default: new.\")\n p_list.add_argument(\"--provider\")\n p_list.add_argument(\"--owner\")\n\n p_show = sub.add_parser(\"show\", help=\"Reveal full token for a finding.\")\n p_show.add_argument(\"id\", type=int)\n\n p_mark = sub.add_parser(\"mark\", help=\"Update finding status.\")\n p_mark.add_argument(\"id\", type=int)\n p_mark.add_argument(\"status\", choices=VALID_STATUSES)\n\n sub.add_parser(\"owners\", help=\"Group new findings by owner (email worklist).\")\n\n p_cand = sub.add_parser(\n \"candidates\",\n help=\"Provisional discovery patterns seen in N+ distinct owners.\",\n )\n p_cand.add_argument(\"--threshold\", type=int, default=3,\n help=\"Minimum distinct owners to qualify (default 3).\")\n\n p_prom = sub.add_parser(\n \"promote\",\n help=\"Emit a patterns.py Provider snippet for a provisional prefix \"\n \"and mark it as promoted in the store.\",\n )\n p_prom.add_argument(\"prefix\", help=\"Prefix from `candidates` output, e.g. 'xyzprov-'\")\n p_prom.add_argument(\"--report-email\", default=\"\",\n help=\"Security contact to bake into the snippet.\")\n p_prom.add_argument(\"--report-url\", default=\"\",\n help=\"Disclosure URL to bake into the snippet.\")\n\n args = parser.parse_args(argv)\n store = FindingStore(args.db)\n\n if args.cmd == \"scan\":\n return _cmd_scan(args, store)\n if args.cmd == \"list\":\n return _cmd_list(args, store)\n if args.cmd == \"show\":\n return _cmd_show(args, store)\n if args.cmd == \"mark\":\n return _cmd_mark(args, store)\n if args.cmd == \"owners\":\n return _cmd_owners(store)\n if args.cmd == \"candidates\":\n return _cmd_candidates(args, store)\n if args.cmd == \"promote\":\n return _cmd_promote(args, store)\n return 2\n\n\ndef _cmd_scan(args, store: FindingStore) -> int:\n # Load dork file into the query list if provided.\n dork_queries: list[str] = []\n if args.dork_file:\n try:\n for line in args.dork_file.read_text().splitlines():\n q = line.strip()\n if q and not q.startswith(\"#\"):\n dork_queries.append(q)\n except OSError as e:\n print(f\"error reading --dork-file: {e}\", file=sys.stderr)\n return 2\n print(f\"loaded {len(dork_queries)} queries from {args.dork_file}\",\n file=sys.stderr)\n\n reporters = [StdoutReporter()]\n if args.send:\n try:\n reporters.append(EmailReporter(\n smtp_host=os.environ[\"LEAKGUARD_SMTP_HOST\"],\n smtp_port=int(os.environ.get(\"LEAKGUARD_SMTP_PORT\", \"587\")),\n smtp_user=os.environ[\"LEAKGUARD_SMTP_USER\"],\n smtp_password=os.environ[\"LEAKGUARD_SMTP_PASSWORD\"],\n from_addr=os.environ[\"LEAKGUARD_FROM\"],\n ))\n except KeyError as e:\n print(f\"error: --send requires env var {e.args[0]}\", file=sys.stderr)\n return 2\n\n # One session shared across sources — cross-source learning AND\n # persistence of learned patterns to the DB for voting/promotion.\n session = DiscoverySession(store=store)\n\n total = 0\n if args.source in (\"github\", \"all\"):\n try:\n gh = GitHubSource(discovery_session=session)\n except RuntimeError as e:\n print(f\"error: {e}\", file=sys.stderr)\n return 2\n total += Pipeline(\n sources=[gh], reporters=reporters,\n store=store, dry_run=not args.send,\n ).run(args.query or dork_queries or list(DEFAULT_GITHUB_QUERIES),\n limit_per_query=args.limit)\n\n if args.source in (\"gitlab\", \"all\"):\n try:\n gl = GitLabSource(\n group_id=args.gitlab_group,\n project_id=args.gitlab_project,\n discovery_session=session,\n )\n except RuntimeError as e:\n print(f\"error: {e}\", file=sys.stderr)\n return 2\n if not (args.gitlab_group or args.gitlab_project):\n print(\"gitlab: no --gitlab-group/--gitlab-project set; \"\n \"global search needs Premium. Skipping unscoped query.\",\n file=sys.stderr)\n else:\n try:\n total += Pipeline(\n sources=[gl], reporters=reporters,\n store=store, dry_run=not args.send,\n ).run(args.query or list(DEFAULT_GITLAB_QUERIES),\n limit_per_query=args.limit)\n except RuntimeError as e:\n print(f\"gitlab: {e}\", file=sys.stderr)\n\n if args.source in (\"google\", \"all\"):\n total += Pipeline(\n sources=[GoogleSource(discovery_session=session)],\n reporters=reporters,\n store=store, dry_run=not args.send,\n ).run(args.query or list(DEFAULT_GOOGLE_QUERIES),\n limit_per_query=args.limit)\n\n if args.source == \"github_events\":\n try:\n ev = GitHubEventsSource(discovery_session=session)\n except RuntimeError as e:\n print(f\"error: {e}\", file=sys.stderr)\n return 2\n # Events source ignores queries — it tails the firehose.\n # `--limit` caps events per poll; call repeatedly for a live feed.\n total += Pipeline(\n sources=[ev], reporters=reporters,\n store=store, dry_run=not args.send,\n ).run([\"\"], limit_per_query=args.limit)\n\n print(f\"{total} new finding(s).\", file=sys.stderr)\n return 0\n\n\ndef _cmd_list(args, store: FindingStore) -> int:\n status = None if args.status == \"all\" else args.status\n print(f\"{'ID':<5} {'PROV':<14} {'STATUS':<14} {'OWNER':<18} \"\n f\"{'AGE':<7} {'FLAGS':<8} {'LHS_HINT':<26} TOKEN(redacted)\")\n for row in store.list(status=status, provider=args.provider, owner=args.owner):\n age = _age_from_meta(row.metadata)\n flags = _flags_from_meta(row.metadata)\n hint = (row.lhs_hint or \"-\")[:26]\n print(\n f\"{row.id:<5} {row.provider[:14]:<14} {row.status:<14} \"\n f\"{(row.owner or '-')[:18]:<18} \"\n f\"{age:<7} {flags[:8]:<8} {hint:<26} \"\n f\"{row.token[:8]}...{row.token[-4:]}\"\n )\n return 0\n\n\ndef _cmd_show(args, store: FindingStore) -> int:\n row = store.get(args.id)\n if not row:\n print(f\"no finding with id {args.id}\", file=sys.stderr)\n return 1\n print(f\"id: {row.id}\")\n print(f\"provider: {row.provider}\")\n print(f\"source: {row.source}\")\n print(f\"location: {row.location}\")\n print(f\"owner: {row.owner or '-'}\")\n print(f\"status: {row.status}\")\n print(f\"first_seen: {row.first_seen}\")\n print(f\"last_seen: {row.last_seen}\")\n print(f\"reported: {row.report_sent_at or '-'}\")\n print(f\"lhs_hint: {row.lhs_hint or '-'}\")\n if row.metadata:\n print(\"repo:\")\n for k, v in row.metadata.items():\n if v is None or v == \"\":\n continue\n print(f\" {k}: {v}\")\n print(f\"context: {row.context}\")\n print(f\"token: {row.token}\")\n return 0\n\n\ndef _age_from_meta(meta: dict) -> str:\n ts = meta.get(\"pushed_at\") or meta.get(\"updated_at\")\n if not ts:\n return \"-\"\n try:\n from datetime import datetime, timezone\n dt = datetime.fromisoformat(ts.replace(\"Z\", \"+00:00\"))\n days = (datetime.now(timezone.utc) - dt).days\n except (ValueError, AttributeError):\n return \"-\"\n if days < 1:\n return \"today\"\n if days < 30:\n return f\"{days}d\"\n if days < 365:\n return f\"{days // 30}mo\"\n return f\"{days // 365}y\"\n\n\ndef _flags_from_meta(meta: dict) -> str:\n flags = []\n if meta.get(\"archived\"):\n flags.append(\"arch\")\n if meta.get(\"fork\"):\n flags.append(\"fork\")\n if (meta.get(\"stars\") or 0) >= 100:\n flags.append(f\"★{meta['stars']}\")\n if meta.get(\"owner_type\") == \"Organization\":\n flags.append(\"org\")\n return \",\".join(flags) or \"-\"\n\n\ndef _cmd_mark(args, store: FindingStore) -> int:\n if not store.get(args.id):\n print(f\"no finding with id {args.id}\", file=sys.stderr)\n return 1\n store.set_status(args.id, args.status)\n msg = f\"id {args.id} -> {args.status}\"\n # Marking a finding false_positive auto-suppresses future ones with\n # the same shape (provider + token prefix + LHS hint).\n if args.status == \"false_positive\":\n if store.add_suppression_from(args.id):\n msg += \" (suppression rule added; future matches auto-FP)\"\n print(msg)\n return 0\n\n\ndef _cmd_owners(store: FindingStore) -> int:\n for owner, rows in store.by_owner():\n print(f\"\\n== {owner} ({len(rows)} finding(s)) ==\")\n for row in rows:\n print(f\" [{row.id}] {row.provider:<10} {row.location}\")\n return 0\n\n\ndef _cmd_candidates(args, store: FindingStore) -> int:\n rows = store.promotion_candidates(threshold=args.threshold)\n if not rows:\n print(f\"no candidates with ≥{args.threshold} distinct owners yet.\",\n file=sys.stderr)\n return 0\n print(f\"{'PREFIX':<20} {'OWNERS':<7} {'HITS':<6} FIRST_SEEN EXAMPLE\")\n for r in rows:\n print(\n f\"{r['prefix']:<20} {r['distinct_owners']:<7} {r['hit_count']:<6} \"\n f\"{r['first_seen']:<22} {r['example'][:12]}...{r['example'][-4:]}\"\n )\n print(\n f\"\\nTo promote: python -m leakguard.cli promote <prefix> \"\n \"--report-email <email>\",\n file=sys.stderr,\n )\n return 0\n\n\ndef _cmd_promote(args, store: FindingStore) -> int:\n rows = [r for r in store.load_provisional_patterns(include_promoted=True)\n if r[\"prefix\"] == args.prefix]\n if not rows:\n print(f\"no provisional pattern for prefix {args.prefix!r}\",\n file=sys.stderr)\n return 1\n row = rows[0]\n snippet = _provider_snippet(\n prefix=row[\"prefix\"],\n safe_name=row[\"safe_name\"],\n regex=row[\"regex\"],\n report_email=args.report_email,\n report_url=args.report_url,\n )\n print(\"# Paste into patterns.py inside the PROVIDERS tuple:\")\n print(snippet)\n store.mark_promoted(args.prefix)\n print(f\"\\n# marked {args.prefix!r} as promoted in the store.\",\n file=sys.stderr)\n return 0\n\n\ndef _provider_snippet(\n prefix: str, safe_name: str, regex: str,\n report_email: str, report_url: str,\n) -> str:\n email = f'\"{report_email}\"' if report_email else '\"\"'\n url = f'\"{report_url}\"' if report_url else \"None\"\n return (\n f\" Provider(\\n\"\n f\" name={safe_name!r},\\n\"\n f\" pattern=re.compile(r{regex!r}),\\n\"\n f\" report_email={email},\\n\"\n f\" report_url={url},\\n\"\n f\" notes=\\\"Promoted from provisional discovery prefix {prefix!r}.\\\",\\n\"\n f\" ),\"\n )\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n"}}}} |