"""Step 3 (repeat, November 2026; same rules as October): compare the server HTML (what a crawler that doesn't run JavaScript gets)
with the rendered DOM (what Chromium shows after JavaScript) for every site that
rendered. The same parser reads both, so the comparison is like for like.

Per version it records: <title>, meta description, rel=canonical, first <h1>,
number of unique internal <a href> links, and the length of the text in <body>
(non-space characters, without script/style/noscript/template/svg).
Writes sites.csv and summary.json.
"""
import csv, gzip, json, os, re
from html.parser import HTMLParser
from urllib.parse import urljoin, urlsplit, urldefrag

SKIP = {"script", "style", "noscript", "template", "svg", "head"}
CONSENT = re.compile(r"consent|cookie|privacy|privacidad|datenschutz|voorkeuren|confidentialit", re.I)
WALL = re.compile(r"robot|captcha|access denied|just a moment|attention required|forbidden", re.I)
VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"}

class Page(HTMLParser):
    def __init__(self, base):
        super().__init__(convert_charrefs=True)
        self.base = base; self.title = None; self.desc = None; self.canonical = None
        self.h1 = None; self.h1_count = 0; self.links = set(); self.chars = 0
        self.stack = []; self._title = None; self._h1 = None; self.in_body = False

    def handle_starttag(self, tag, attrs):
        a = {k.lower(): (v or "") for k, v in attrs}
        if tag == "body": self.in_body = True
        if tag == "title" and self.title is None and "svg" not in self.stack: self._title = []
        if tag == "meta" and a.get("name", "").lower() == "description" and self.desc is None:
            self.desc = a.get("content", "").strip()
        if tag == "link" and "canonical" in a.get("rel", "").lower().split() and self.canonical is None:
            self.canonical = a.get("href", "").strip()
        if tag == "h1":
            self.h1_count += 1
            if self.h1 is None and self._h1 is None: self._h1 = []
        if tag == "a" and a.get("href"):
            h = a["href"].strip()
            if not h.startswith(("#", "javascript:", "mailto:", "tel:")):
                u = urldefrag(urljoin(self.base, h))[0]
                if same_site(u, self.base): self.links.add(u.rstrip("/"))
        if tag not in VOID: self.stack.append(tag)

    def handle_endtag(self, tag):
        if tag == "title" and self._title is not None:
            self.title = " ".join("".join(self._title).split()); self._title = None
        if tag == "h1" and self._h1 is not None:
            t = " ".join("".join(self._h1).split()); self._h1 = None
            if t and not CONSENT.search(t): self.h1 = t  # skip H1s of cookie/consent dialogs
        if tag in self.stack:
            while self.stack and self.stack.pop() != tag: pass

    def handle_data(self, d):
        if self._title is not None: self._title.append(d)
        if self._h1 is not None: self._h1.append(d)
        if self.in_body and not (SKIP & set(self.stack)):
            self.chars += sum(1 for c in d if not c.isspace())

def host(u):
    h = (urlsplit(u).hostname or "").lower()
    return h[4:] if h.startswith("www.") else h

def same_site(u, base):
    if urlsplit(u).scheme not in ("http", "https"): return False
    a, b = host(u), host(base)
    return a == b or a.endswith("." + b) or b.endswith("." + a)

def parse(path, base):
    p = Page(base)
    p.feed(gzip.open(path, "rt", encoding="utf-8", errors="replace").read()); p.close()
    return {"title": p.title or "", "desc": p.desc or "", "canonical": p.canonical or "",
            "h1": p.h1 or "", "h1_count": p.h1_count, "links": len(p.links), "chars": p.chars}

sel = {s["domain"]: s for s in json.load(open("selected.json"))}
ren = [json.loads(l) for l in open("render.jsonl")]
rows = []
for r in ren:
    if not r.get("ok"): continue
    s = sel[r["domain"]]
    if not os.path.exists(f"raw/{r['domain']}.html.gz"):
        rows.append({"domain": r["domain"], "rank": s["rank"], "framework": s["frameworks"][0], "excluded": "server HTML fetch failed (no 200 HTML)"}); continue
    raw = parse(f"raw/{r['domain']}.html.gz", s["final_url"])
    dom = parse(f"rendered/{r['domain']}.html.gz", r.get("final_url") or s["final_url"])
    why = ""
    if WALL.search(dom["title"]): why = "the browser got a bot check instead of the page"
    elif dom["chars"] < 200: why = "rendered page under 200 characters (bot, consent or error page)"
    if why:
        rows.append({"domain": r["domain"], "rank": s["rank"], "framework": s["frameworks"][0], "excluded": why}); continue
    gone = lambda k: bool(dom[k]) and not raw[k]
    text_share = raw["chars"] / dom["chars"]
    link_share = raw["links"] / dom["links"] if dom["links"] else 1.0
    row = {
        "domain": r["domain"], "rank": s["rank"], "framework": s["frameworks"][0], "excluded": "",
        "raw_chars": raw["chars"], "rendered_chars": dom["chars"], "text_share": round(text_share, 3),
        "raw_links": raw["links"], "rendered_links": dom["links"], "link_share": round(link_share, 3),
        "title_missing_raw": gone("title"), "title_changed": bool(raw["title"] and dom["title"] and raw["title"] != dom["title"]),
        "desc_missing_raw": gone("desc"), "canonical_missing_raw": gone("canonical"),
        "canonical_changed": bool(raw["canonical"] and dom["canonical"] and raw["canonical"].rstrip("/") != dom["canonical"].rstrip("/")),
        "h1_missing_raw": gone("h1"),
    }
    row["empty_shell"] = text_share < 0.10
    row["text_lost_half"] = text_share < 0.5
    row["links_lost_half"] = link_share < 0.5
    row["loses_important"] = any(row[k] for k in ("title_missing_raw", "h1_missing_raw", "text_lost_half", "links_lost_half", "desc_missing_raw", "canonical_missing_raw"))
    rows.append(row)

cols = ["domain", "rank", "framework", "excluded", "raw_chars", "rendered_chars", "text_share", "raw_links", "rendered_links", "link_share",
        "title_missing_raw", "title_changed", "desc_missing_raw", "canonical_missing_raw", "canonical_changed", "h1_missing_raw",
        "empty_shell", "text_lost_half", "links_lost_half", "loses_important"]
with open("sites.csv", "w", newline="") as f:
    w = csv.DictWriter(f, cols); w.writeheader()
    for r in rows: w.writerow({c: r.get(c, "") for c in cols})

ok = [r for r in rows if not r["excluded"]]
n = len(ok)
cnt = lambda k, rs=ok: sum(1 for r in rs if r[k])
SSR = {"Next.js", "Nuxt", "SvelteKit", "Gatsby", "Remix"}  # frameworks that render on the server by default
ssr = [r for r in ok if r["framework"] in SSR]; csr = [r for r in ok if r["framework"] not in SSR]
by_fw = {}
for r in ok: by_fw.setdefault(r["framework"], []).append(r)
summary = {
    "repeat_of": "javascript-seo-2026-10-06 (same 100 sites, selected.json unchanged)",
    "raw_fetched_ok": sum(1 for l in open("fetch.jsonl") if json.loads(l).get("ok")),
    "selected": len(sel), "rendered_ok": sum(1 for r in ren if r.get("ok")),
    "excluded_after_render": len(rows) - n, "compared": n,
    "loses_important": cnt("loses_important"), "empty_shell": cnt("empty_shell"),
    "text_lost_half": cnt("text_lost_half"), "links_lost_half": cnt("links_lost_half"),
    "title_missing_raw": cnt("title_missing_raw"), "title_changed": cnt("title_changed"),
    "desc_missing_raw": cnt("desc_missing_raw"), "canonical_missing_raw": cnt("canonical_missing_raw"),
    "canonical_changed": cnt("canonical_changed"), "h1_missing_raw": cnt("h1_missing_raw"),
    "raw_has_no_internal_links": sum(1 for r in ok if r["raw_links"] == 0 and r["rendered_links"] > 0),
    "server_rendering_frameworks": {"n": len(ssr), "loses_important": cnt("loses_important", ssr), "empty_shell": cnt("empty_shell", ssr)},
    "client_side_frameworks": {"n": len(csr), "loses_important": cnt("loses_important", csr), "empty_shell": cnt("empty_shell", csr)},
    "by_framework": {k: {"n": len(v), "loses_important": cnt("loses_important", v), "empty_shell": cnt("empty_shell", v)} for k, v in sorted(by_fw.items(), key=lambda x: -len(x[1]))},
}
json.dump(summary, open("summary.json", "w"), indent=1)
print(json.dumps(summary, indent=1))
