#!/usr/bin/env python3"""One command rebuilds everything.  python3 run.py --offline   rebuild every number and chart from the committed snapshots, then verify them  python3 run.py             pull fresh keyless data first, then rebuild  python3 run.py --bq-light  also refresh the two rolling BigQuery queries, about 5 GB (skipped without credentials)  python3 run.py --bq        also refresh the larger BigQuery snapshots (needs GOOGLE_APPLICATION_CREDENTIALS, free sandbox works)  python3 run.py --explorer  also rebuild swap aggregates from local Explorer rowsOffline needs Python 3.9+ and nothing else. Node is used only by the model parity test, which is skipped if absent."""import argparse, gzip, hashlib, json, os, pathlib, shutil, subprocess, sys, time, zlibROOT = pathlib.Path(__file__).resolve().parentPY = sys.executabledef step(name, *args, ok_codes=(0,)):    t0 = time.time()    res = subprocess.run([PY, *map(str, args)], cwd=ROOT)    print(f"[{name}] exit {res.returncode}, {time.time() - t0:.1f}s", flush=True)    if res.returncode not in ok_codes:        raise SystemExit(f"step failed: {name}")def content_hash(path):    """sha256 of a snapshot's decompressed content, or None if the file is damaged."""    try:        with gzip.open(path, "rb") as fh:            return hashlib.sha256(fh.read()).hexdigest()    except (OSError, EOFError, zlib.error):        return Nonedef verify_snapshots():    """Every committed snapshot must hash to its newest manifest entry."""    manifest = ROOT / "raw" / "MANIFEST.jsonl"    latest = {}    for line in manifest.read_text().splitlines():        rec = json.loads(line)        latest[rec["name"]] = rec    checked = bad = 0    for name, rec in latest.items():        path = ROOT / "raw" / "latest" / f"{name}.json.gz"        if not path.exists():            continue        got = content_hash(path)        checked += 1        if got != rec["sha256"]:            bad += 1            print(f"HASH MISMATCH {name}")    bq = ROOT / "raw" / "bigquery" / "MANIFEST.jsonl"    latest_bq = {}    for line in bq.read_text().splitlines():        rec = json.loads(line)        latest_bq[rec["file"]] = rec    for name, rec in latest_bq.items():        got = content_hash(ROOT / "raw" / "bigquery" / name)        checked += 1        if got != rec["sha256_csv"]:            bad += 1            print(f"HASH MISMATCH {name}")    ex = ROOT / "raw" / "explorer" / "MANIFEST.json"    if ex.exists():        for rec in json.loads(ex.read_text())["files"]:            got = content_hash(ROOT / "raw" / "explorer" / rec["file"])            checked += 1            if got != rec["sha256_csv"]:                bad += 1                print(f"HASH MISMATCH {rec['file']}")    print(f"snapshots verified: {checked} ok, {bad} bad")    if bad:        raise SystemExit("raw snapshots do not match the manifest")def main():    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)    ap.add_argument("--offline", action="store_true")    ap.add_argument("--bq", action="store_true")    ap.add_argument("--bq-light", action="store_true")    ap.add_argument("--explorer", action="store_true")    args = ap.parse_args()    if not args.offline:        step("public data", "pipeline/fetch_public.py", ok_codes=(0, 1))        step("chain data", "pipeline/fetch_chain.py", ok_codes=(0, 1))    if args.bq:        step("bigquery", "pipeline/run_bq.py", "--only", "01,02,09,10,12,13", "--cap-gb", "120", ok_codes=(0, 1))    if args.bq_light:        if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):            step("bigquery rolling", "pipeline/run_bq.py", "--only", "01b,09b", "--cap-gb", "12", ok_codes=(0, 1))        else:            print("[bigquery rolling] skipped: no credentials, using committed snapshots")    if args.explorer:        step("explorer aggregates", "pipeline/analyze_explorer.py")    verify_snapshots()    step("analysis", "pipeline/analyze.py")    step("model", "pipeline/model.py")    step("kill criteria", "pipeline/grade.py", *(["--offline"] if args.offline else []))    step("site", "pipeline/build.py")    step("claims", "pipeline/verify_claims.py")    tests = ["tests/test_pipeline.py"]    if shutil.which("node"):        tests.append("tests/test_model.py")    for t in tests:        step(t, t)    print("done: open docs/index.html through any static server, e.g. python3 -m http.server -d docs 8000")if __name__ == "__main__":    main()