1 line
4.6 KiB
JSON
1 line
4.6 KiB
JSON
{"repo": "acoliver/vibetools", "files": {"research/ai-code-review-study/ocr/reliability/validation.py": {"d57789dc2c17f2be39efe2c437f25ea98457fae8": {"t": 1785674636.061543, "n": 4412, "h": "62227b88f50ff3ca", "c": "#!/usr/bin/env python3\n\"\"\"Targeted validation for the OCR reliability research package.\"\"\"\nimport csv, hashlib, re, subprocess, sys\nfrom collections import Counter\nfrom pathlib import Path\n\nBASE=Path(__file__).resolve().parent\nROOT=BASE.parents[1]\nrequired=['report.md','run-events.csv','provider-timeline.csv','reliability-by-period.csv','chart-timeseries.csv','reliability-over-time.svg','reliability-over-time.png','generate-reliability-chart.py','evidence-index.md','methodology.md','commands.md','source-extracts.md','cost-evidence.md','file-inventory.txt','validation.py']\nerrors=[]\ndef check(ok,msg):\n print(('PASS' if ok else 'FAIL'),msg)\n if not ok: errors.append(msg)\n\ncheck(all((BASE/x).exists() for x in required),'all 15 required artifacts exist')\nwith (BASE/'run-events.csv').open(newline='') as f:\n rd=csv.DictReader(f); events=list(rd); check(len(rd.fieldnames)==len(set(rd.fieldnames)),'run-events headers unique')\ncheck(len(events)==145,'run-events has 145 rows')\ncheck(len({r['event_id'] for r in events})==145,'event IDs unique')\ncheck(len({tuple(r['source_path'].split(';')) for r in events})==145,'canonical source sets unique')\nout=Counter(r['outcome'] for r in events)\ncheck(out=={'success':117,'partial':25,'failure':3},f'outcome aggregate exact: {dict(out)}')\nprov=Counter((r['provider'],r['outcome']) for r in events)\nexpected={('Z.ai','success'):66,('Z.ai','partial'):13,('Z.ai','failure'):1,('StepFun','success'):26,('StepFun','partial'):12,('StepFun','failure'):2,('unknown','success'):25}\ncheck(prov==expected,f'provider aggregate exact: {dict(prov)}')\ncheck(sum('HTTP 429' in r['notes'] for r in events)==19,'19 partial HTTP 429 causes')\ncheck(sum('HTTP 529' in r['notes'] for r in events)==6,'6 partial HTTP 529 causes')\nfor name,n in [('provider-timeline.csv',8),('reliability-by-period.csv',7),('chart-timeseries.csv',5)]:\n with (BASE/name).open(newline='') as f: rows=list(csv.DictReader(f))\n check(len(rows)==n,f'{name} has {n} data rows')\nif (BASE/'reliability-by-period.csv').exists():\n pr=list(csv.DictReader((BASE/'reliability-by-period.csv').open()))\n check(sum(int(r['attempts']) for r in pr)==145,'period attempts sum to canonical events')\n check(sum(int(r['successes']) for r in pr)==117 and sum(int(r['partials']) for r in pr)==25 and sum(int(r['hard_failures']) for r in pr)==3,'period outcomes sum exactly')\nif (BASE/'chart-timeseries.csv').exists():\n cr=list(csv.DictReader((BASE/'chart-timeseries.csv').open()))\n check(sum(int(r['attempts']) for r in cr)==145,'chart attempts sum to canonical events')\n check(sum(int(r['successes']) for r in cr)==117 and sum(int(r['partials']) for r in cr)==25 and sum(int(r['failures']) for r in cr)==3,'chart outcomes sum exactly')\ncheck((BASE/'reliability-over-time.svg').read_text().lstrip().startswith('<svg'),'SVG has SVG root')\ncheck((BASE/'reliability-over-time.png').read_bytes().startswith(b'\\x89PNG\\r\\n\\x1a\\n'),'PNG has valid signature')\n# Relative Markdown links within package and parent research tree.\nfor p in BASE.glob('*.md'):\n text=p.read_text()\n for link in re.findall(r'\\]\\(([^)]+)\\)',text):\n if link.startswith(('http://','https://','#')): continue\n check((p.parent/link.split('#')[0]).resolve().exists(),f'relative link exists: {p.name} -> {link}')\n# Credential scan: long opaque values and common secret/header patterns.\nscan='\\n'.join(p.read_text(errors='ignore') for p in BASE.iterdir() if p.suffix not in ('.png',) and p.name!='file-inventory.txt')\npatterns=[r'(?i)authorization\\s*:\\s*bearer\\s+[A-Za-z0-9._-]{8,}',r'(?i)(?:api[_ -]?key|token|secret|password)\\s*[=:]\\s*[\"\\x27]?[A-Za-z0-9._-]{16,}',r'\\bsk-[A-Za-z0-9_-]{12,}\\b']\ncheck(not any(re.search(x,scan) for x in patterns),'credential scan clean')\n# Inventory hashes every final file except inventory itself.\ninv={}\nif (BASE/'file-inventory.txt').exists():\n for line in (BASE/'file-inventory.txt').read_text().splitlines()[1:]:\n if not line: continue\n path,size,sha=line.split('\\t'); inv[path]=(int(size),sha)\nfor p in sorted(BASE.iterdir()):\n if not p.is_file() or p.name in ('file-inventory.txt','.harvest.py','.finalize.py'): continue\n rel=str(p.relative_to(ROOT)); got=(p.stat().st_size,hashlib.sha256(p.read_bytes()).hexdigest())\n check(inv.get(rel)==got,f'inventory matches {p.name}')\nprint(f'RESULT checks_complete errors={len(errors)}')\nsys.exit(bool(errors))\n"}}}} |