"""Step 1: find popular sites built with a JavaScript framework.

Sample: Tranco list 94GG2 (tranco-list.eu), ranks 1-100 000, random.Random(20261006).
For each domain the script makes ONE request to https://<domain>/ (curl follows
redirects, desktop Chrome user agent, 15 s limit), keeps the HTML the server sends
and looks for framework fingerprints in it. The raw HTML is kept locally in raw/
(not published) so step 2 compares against exactly the bytes a non-JS crawler gets.
"""
import csv, gzip, json, os, random, re, subprocess, sys, time
from concurrent.futures import ThreadPoolExecutor

UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"
N = int(sys.argv[1]) if len(sys.argv) > 1 else 1200
os.makedirs("raw", exist_ok=True)

MARKERS = [  # (framework, regex on raw HTML)
    ("Next.js", r"__NEXT_DATA__|/_next/static/|self\.__next_f"),
    ("Nuxt", r"__NUXT__|__NUXT_DATA__|/_nuxt/|data-n-head"),
    ("Gatsby", r"id=\"___gatsby\"|/page-data/app-data"),
    ("SvelteKit", r"__sveltekit|/_app/immutable/"),
    ("Remix", r"__remixContext|__remixManifest"),
    ("Angular", r"ng-version=|<app-root"),
    ("Vue", r"data-v-[0-9a-f]{8}|data-server-rendered|chunk-vendors\.[0-9a-f]+\.js"),
    ("React", r"data-reactroot|/static/js/main\.[0-9a-f]+\.(chunk\.)?js|<div id=\"root\">\s*</div>|react-dom"),
]

def fetch(domain):
    url = f"https://{domain}/"
    r = subprocess.run(["curl", "-sL", "--compressed", "--max-time", "15", "-A", UA,
                        "-H", "Accept: text/html,application/xhtml+xml",
                        "-H", "Accept-Language: en-US,en;q=0.9",
                        "-w", "\n__META__%{http_code}\t%{url_effective}\t%{content_type}", url],
                       capture_output=True)
    out = r.stdout.decode("utf-8", "replace")
    body, _, meta = out.rpartition("\n__META__")
    code, final, ctype = (meta.split("\t") + ["", "", ""])[:3]
    rec = {"domain": domain, "code": int(code or 0), "final_url": final, "ctype": ctype}
    if rec["code"] == 200 and "html" in ctype.lower() and len(body) > 200:
        rec["frameworks"] = [name for name, rx in MARKERS if re.search(rx, body)]
        rec["raw_bytes"] = len(body.encode())
        with gzip.open(f"raw/{domain}.html.gz", "wt", encoding="utf-8") as f:
            f.write(body)
    return rec

rows = [r[1] for r in csv.reader(open("top-1m.csv")) if int(r[0]) <= 100_000]
rank = {r[1]: int(r[0]) for r in csv.reader(open("top-1m.csv")) if int(r[0]) <= 100_000}
sample = random.Random(20261006).sample(rows, N)
t0 = time.time()
with ThreadPoolExecutor(24) as ex, open("screen.jsonl", "w") as out:
    for i, rec in enumerate(ex.map(fetch, sample), 1):
        rec["rank"] = rank[rec["domain"]]; rec["order"] = i
        out.write(json.dumps(rec) + "\n")
        if i % 200 == 0: print(i, round(time.time() - t0), flush=True)
print("done", N, round(time.time() - t0))
