#!/usr/bin/env python3
"""Count WCAG 2.0 / 2.1 / 2.2 success criteria by level, principle and guideline.

Reads the W3C Recommendations (https://www.w3.org/TR/WCAG20/, /WCAG21/, /WCAG22/),
extracts every success criterion with its conformance level and writes:

  criteria-counts.json  - totals, per-level, per-principle, diffs between versions
  criteria.csv          - one row per success criterion per version

Standard library only. Usage:

  python3 count_criteria.py                 # fetch the three documents
  python3 count_criteria.py --cache DIR     # read DIR/WCAG20.html etc. if present,
                                            # otherwise fetch and store them there
"""
import argparse, csv, hashlib, html, json, re, sys, urllib.request
from datetime import datetime, timezone
from pathlib import Path

HERE = Path(__file__).resolve().parent
URLS = {v: f"https://www.w3.org/TR/WCAG{v.replace('.', '')}/" for v in ("2.0", "2.1", "2.2")}
PRINCIPLES = {"1": "Perceivable", "2": "Operable", "3": "Understandable", "4": "Robust"}
UA = "CometWeb research script (hello@cometweb.io)"


def load(version, cache):
    name = f"WCAG{version.replace('.', '')}.html"
    if cache and (cache / name).exists():
        return (cache / name).read_bytes()
    req = urllib.request.Request(URLS[version], headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=60) as r:
        body = r.read()
    if cache:
        cache.mkdir(parents=True, exist_ok=True)
        (cache / name).write_bytes(body)
    return body


def strip(s):
    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", "", s))).strip()


def parse_20(text):
    # <strong class="sc-handle">1.1.1 Non-text Content:</strong> ... (Level A)
    out = []
    handles = list(re.finditer(r'<strong class="sc-handle">\s*(\d\.\d+\.\d+)\s+([^<]+?):?\s*</strong>', text))
    for i, m in enumerate(handles):
        end = handles[i + 1].start() if i + 1 < len(handles) else len(text)
        lvl = re.search(r"\(Level (A{1,3})\)", text[m.end():end])
        out.append({"sc": m.group(1), "name": strip(m.group(2)).rstrip(":"),
                    "level": lvl.group(1) if lvl else None, "status": "active"})
    return out


def parse_2x(text):
    # <bdi class="secno">Success Criterion 1.1.1 </bdi>Non-text Content</h4> ... <p class="conformance-level">Level A</p>
    out = []
    heads = list(re.finditer(r'<bdi class="secno">Success Criterion (\d\.\d+\.\d+)\s*</bdi>(.*?)</h4>', text, re.S))
    for i, m in enumerate(heads):
        end = heads[i + 1].start() if i + 1 < len(heads) else len(text)
        seg = text[m.end():end]
        lvl = re.search(r'class="conformance-level"[^>]*>\s*\(?Level (A{1,3})\)?', seg)
        name = strip(m.group(2))
        obsolete = "obsolete and removed" in name.lower()
        out.append({"sc": m.group(1), "name": re.sub(r"\s*\(Obsolete and removed\)", "", name),
                    "level": lvl.group(1) if lvl else None,
                    "status": "removed" if obsolete else "active"})
    return out


def key(sc):
    return tuple(int(x) for x in sc.split("."))


def summarise(rows):
    active = [r for r in rows if r["status"] == "active"]
    by_level = {l: sum(1 for r in active if r["level"] == l) for l in ("A", "AA", "AAA")}
    by_principle = {}
    for p, pname in PRINCIPLES.items():
        pr = [r for r in active if r["sc"].split(".")[0] == p]
        by_principle[pname] = {
            "A": sum(1 for r in pr if r["level"] == "A"),
            "AA": sum(1 for r in pr if r["level"] == "AA"),
            "AAA": sum(1 for r in pr if r["level"] == "AAA"),
            "total": len(pr),
        }
    guidelines = sorted({".".join(r["sc"].split(".")[:2]) for r in active}, key=key)
    return {
        "total_active": len(active),
        "by_level": by_level,
        "a_plus_aa": by_level["A"] + by_level["AA"],
        "by_principle": by_principle,
        "guidelines": len(guidelines),
        "removed": [r["sc"] for r in rows if r["status"] == "removed"],
        "unlabelled": [r["sc"] for r in active if r["level"] is None],
    }


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--cache", type=Path)
    args = ap.parse_args()
    data, docs = {}, {}
    for v in URLS:
        raw = load(v, args.cache)
        docs[v] = {"url": URLS[v], "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
        text = raw.decode("utf-8", "replace")
        rows = parse_20(text) if v == "2.0" else parse_2x(text)
        rows.sort(key=lambda r: key(r["sc"]))
        data[v] = rows

    def active(v):
        return {r["sc"]: r for r in data[v] if r["status"] == "active"}

    diffs = {}
    for old, new in (("2.0", "2.1"), ("2.1", "2.2")):
        added = sorted(set(active(new)) - set(active(old)), key=key)
        removed = sorted(set(active(old)) - set(active(new)), key=key)
        changed = sorted([s for s in set(active(old)) & set(active(new))
                          if active(old)[s]["level"] != active(new)[s]["level"]], key=key)
        diffs[f"{old}->{new}"] = {
            "added": [{"sc": s, "name": active(new)[s]["name"], "level": active(new)[s]["level"]} for s in added],
            "added_by_level": {l: sum(1 for s in added if active(new)[s]["level"] == l) for l in ("A", "AA", "AAA")},
            "removed": [{"sc": s, "name": active(old)[s]["name"], "level": active(old)[s]["level"]} for s in removed],
            "level_changed": [{"sc": s, "from": active(old)[s]["level"], "to": active(new)[s]["level"]} for s in changed],
        }

    result = {
        "schema": "cometweb.wcag-criteria-count/v1",
        "generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
        "method": "Regex extraction of success-criterion headings and their conformance-level labels from the W3C Recommendation HTML. A criterion marked 'Obsolete and removed' is counted as removed, not active.",
        "documents": docs,
        "summary": {v: summarise(data[v]) for v in data},
        "diffs": diffs,
    }
    (HERE / "criteria-counts.json").write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n")
    with open(HERE / "criteria.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["version", "sc", "name", "level", "principle", "guideline", "status"])
        for v, rows in data.items():
            for r in rows:
                w.writerow([v, r["sc"], r["name"], r["level"] or "", PRINCIPLES[r["sc"][0]],
                            ".".join(r["sc"].split(".")[:2]), r["status"]])
    for v, s in result["summary"].items():
        print(v, s["total_active"], s["by_level"], "A+AA", s["a_plus_aa"], "guidelines", s["guidelines"], "removed", s["removed"], "unlabelled", s["unlabelled"])
    for k, d in diffs.items():
        print(k, "added", len(d["added"]), d["added_by_level"], "removed", [r["sc"] for r in d["removed"]], "level changes", d["level_changed"])


if __name__ == "__main__":
    sys.exit(main())