"""Step 1 (repeat, November 2026): fetch the server HTML again for the same 100 homepages
picked in October (selected.json is copied unchanged from the October pack).
One request per site with curl, redirects followed, 15 s limit, at most 8 at a time.
User agent identifies the research: CometWebResearch/1.0 (+https://cometweb.io).
October used a desktop Chrome user agent, which is the one difference in method.
Output: raw/<domain>.html.gz + fetch.jsonl"""
import gzip, json, subprocess
from concurrent.futures import ThreadPoolExecutor
UA = "CometWebResearch/1.0 (+https://cometweb.io)"
sel = json.load(open("selected.json"))
def fetch(s):
    url = f"https://{s['domain']}/"
    r = subprocess.run(["curl", "-sL", "--compressed", "--max-time", "15", "-A", UA,
                        "-H", "Accept: text/html,application/xhtml+xml", "-H", "Accept-Language: en-US,en;q=0.9",
                        "-w", "\n__META__%{http_code}\t%{url_effective}\t%{content_type}", url], capture_output=True)
    out = r.stdout.decode("utf-8", "replace")
    body, _, meta = out.rpartition("\n__META__")
    code, final, ctype = (meta.split("\t") + ["", "", ""])[:3]
    rec = {"domain": s["domain"], "code": int(code or 0), "final_url": final, "ctype": ctype}
    if rec["code"] == 200 and "html" in ctype.lower() and len(body) > 200:
        rec["raw_bytes"] = len(body.encode()); rec["ok"] = True
        with gzip.open(f"raw/{s['domain']}.html.gz", "wt", encoding="utf-8") as f: f.write(body)
    else:
        rec["ok"] = False
    return rec
with ThreadPoolExecutor(8) as ex, open("fetch.jsonl", "w") as out:
    for rec in ex.map(fetch, sel): out.write(json.dumps(rec) + "\n")
print("fetched", sum(1 for l in open("fetch.jsonl") if '"ok": true' in l), "of", len(sel))
