#!/usr/bin/env python
"""Group endpoint_diff differences by endpoint pattern.

Joins _out/report.json (per-request category/attribution) with _out/manifest.json
(per-request URL), normalises every URL to an endpoint *pattern* -- ids in the
path become `:id`, enum-like query values are kept, dates/numbers dropped --
and reports, per pattern:

  * how many requests to that endpoint were fired and how many differed
  * the difference category mix (ARRAY-ORDER vs CONTENT)
  * whether body length varied (a length change under CONTENT means a
    different *set* of rows came back, not merely a reordering)

Writes report_endpoints.md / report_endpoints.json.
"""
import json
import os
import re
from collections import Counter, defaultdict
from urllib.parse import parse_qsl, urlsplit

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "_out")

_WORD = re.compile(r"[a-z][a-z-]*\Z")
_ENUM = re.compile(r"[A-Za-z][A-Za-z_]*\Z")


def pattern(url):
    """Normalise a URL to an endpoint pattern: path ids -> :id, dates dropped."""
    u = urlsplit(url)
    segs = []
    for seg in u.path.split("/"):
        if seg == "":
            continue
        segs.append(seg if _WORD.match(seg) else ":id")
    path = "/" + "/".join(segs)
    params = []
    for k, v in sorted(parse_qsl(u.query, keep_blank_values=True)):
        params.append(f"{k}={v}" if _ENUM.match(v or "") else k)
    return path + ("?" + "&".join(params) if params else "")


def main():
    manifest = json.load(open(os.path.join(OUT, "manifest.json")))
    report = json.load(open(os.path.join(OUT, "report.json")))
    req_by_id = {r["id"]: r for r in manifest["requests"]}
    row_by_id = {r["id"]: r for r in report["rows"]}

    # all requests grouped by pattern, plus the differing subset
    fired = defaultdict(list)
    for rid, req in req_by_id.items():
        fired[pattern(req["url"])].append(rid)

    groups = {}
    for pat, ids in fired.items():
        diffs = [
            row_by_id[i]
            for i in ids
            if row_by_id.get(i, {}).get("category")
            not in ("IDENTICAL", "MISSING", None)
        ]
        if not diffs:
            continue
        cats = Counter(d["category"] for d in diffs)
        len_changed = sum(1 for d in diffs if d["len_before"] != d["len_after"])
        # a CONTENT diff = the data itself varies (result-set nondeterminism);
        # ARRAY-ORDER = identical data, only row order differs.
        kind = "RESULT-SET" if cats.get("CONTENT") else "ROW-ORDER"
        groups[pat] = dict(
            pattern=pat,
            fired=len(ids),
            differ=len(diffs),
            categories=dict(cats),
            len_changed=len_changed,
            kind=kind,
            example_ids=sorted(d["id"] for d in diffs)[:6],
            example_url=req_by_id[diffs[0]["id"]]["url"],
        )

    ordered = sorted(groups.values(), key=lambda g: (g["kind"], -g["differ"]))
    result_set = [g for g in ordered if g["kind"] == "RESULT-SET"]
    row_order = [g for g in ordered if g["kind"] == "ROW-ORDER"]

    L = []
    a = L.append
    a("# endpoint_diff -- nondeterministic endpoints, grouped by pattern")
    a("")
    a(
        f"{report['identical']} / {report['total']} requests are byte-identical "
        f"before vs after; **{report['differ']}** differ. Grouped below by "
        f"endpoint pattern ({len(groups)} distinct patterns affected). Path ids "
        f"are shown as `:id`; only enum-like query values are kept."
    )
    a("")
    a(
        "- **RESULT-SET** -- the response carries *different data* between two "
        "runs of the same build (a `LIMIT`/top-N over a non-unique ordering "
        "picks a different set of rows). User-visible.\n"
        "- **ROW-ORDER** -- identical data, only the JSON array order differs "
        "(equal once every list is sorted). Cosmetic, but breaks caching, "
        "byte-diffing and ETag stability."
    )
    a("")
    for title, gs in (
        ("RESULT-SET nondeterminism", result_set),
        ("ROW-ORDER nondeterminism", row_order),
    ):
        a(f"## {title} -- {len(gs)} endpoint pattern(s)")
        a("")
        a("| endpoint pattern | differ / fired | categories | len changed |")
        a("|---|---|---|---|")
        for g in gs:
            cats = ", ".join(f"{k}:{v}" for k, v in sorted(g["categories"].items()))
            a(
                f"| `{g['pattern']}` | {g['differ']}/{g['fired']} | {cats} "
                f"| {g['len_changed']} |"
            )
        a("")

    a("## Example URLs")
    a("")
    for g in ordered:
        a(f"- `{g['pattern']}` ({g['kind']})")
        a(
            f"  - e.g. `{urlsplit(g['example_url']).path}?{urlsplit(g['example_url']).query}`"
        )
        a(f"  - ids: {', '.join(g['example_ids'])}")
    a("")

    md = "\n".join(L) + "\n"
    open(os.path.join(OUT, "report_endpoints.md"), "w").write(md)
    json.dump(
        {"result_set": result_set, "row_order": row_order},
        open(os.path.join(OUT, "report_endpoints.json"), "w"),
        indent=2,
        sort_keys=True,
    )
    print(md)
    print(f"wrote {OUT}/report_endpoints.md and report_endpoints.json")


if __name__ == "__main__":
    main()