#!/usr/bin/env python3
"""
ledger.py — THE REGISTRY (ARCHITECTURE §5): a DERIVED coverage oracle over the distributed SoT.

audience: AI coding agents first. This IS "the registry" of §5 (the taxonomy doc calls it the coverage LEDGER).
NOT named registry.py — that would collide with prevent/registry.py (the lower-level detector-MANIFEST loader
this consumes); NOT coverage.py — that shadows the installed `coverage` package (pytest-cov).

WHAT IT DOES: make blind spots structurally impossible to hide — the test that proves the "100%" claim is real.
DERIVES from the distributed SoT, NEVER hand-maintains a parallel one and NEVER executes a detector:
  - cells          — bench.load_cells() (corpus/ + detectors/*/cells/, both locations)
  - capabilities   — detector.json via prevent/registry.load (covers list | class singular | shipped-cell classes)
  - ratchet        — prevent/confirmed.json
  - class set      — docs/taxonomy/security.md, the INDEPENDENT S1–S11 denominator (NOT derived from cells:
                     a zero-cell class is invisible if the denominator comes from the cells — circular)

CI GATE (--check): exit 1 on any STRUCTURAL gap (no-false-coverage-claim made structural):
  - a class with zero cells                      → BLIND SPOT
  - a class with no responsible detector         → UNCOVERED
  - a cell whose class has no responsible detector
  - a ratchet entry whose class has no detector  → INERT RATCHET (false assurance)

GRADE, never a boolean — a flat "11/11 ✓" would ERASE the n=1 tail the taxonomy doc is careful about (that
erasure would itself be a no-false-coverage-claim regression). The CI floor is boolean; the PROJECTION grades:
  - sample   : rate (n>=3 cells) | point-estimate (1-2) | none (0)   — countable, never parsed from prose
  - coverage : deterministic (an exec detector ships a conformance cell of the class) | asserted (an LLM
               `covers` claim — recall measured ELSEWHERE, in bench, NOT proven here) | none
  - discriminator: anti-canary (GREEN-on-safe) grade for the class — 'measured' (k>=3 hash-current rate from
    recall_gate over precision_records.json, rate surfaced), 'present' (safe cell ships, rate UNMEASURED), or 'none'

Recall RATE + conformance EXECUTION stay in bench / the conformance harness. This file does set membership over
classes + dedup; it has no findings, so it does NOT consume verdict.match (§4 lists the registry as Verdict's
TRIGGERING task, not a consumer).
"""
import argparse, glob, importlib.util, json, os, re, sys

ROOT = os.path.dirname(os.path.abspath(__file__))


def _load(name, path):
    spec = importlib.util.spec_from_file_location(name, path)
    m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); return m


bench = _load("sg_bench_cov", os.path.join(ROOT, "bench.py"))            # load_cells (both locations)
manifest = _load("sg_manifest", os.path.join(ROOT, "prevent", "registry.py"))  # detector-manifest loader
recall = _load("sg_recall_gate", os.path.join(ROOT, "recall_gate.py"))  # the rate validator (#5), reused as-is
PRECISION_RECORDS = os.path.join(ROOT, "domains", "security", "recall", "precision_records.json")
TAXONOMY_DOC = os.path.join(ROOT, "docs", "taxonomy", "security.md")
CONFIRMED = os.path.join(ROOT, "prevent", "confirmed.json")


def taxonomy_classes(doc=TAXONOMY_DOC):
    """The INDEPENDENT blind-spot denominator: S1–S11 parsed from the taxonomy ledger TABLE rows (`| Sn | …`).
    Independent of the cells ON PURPOSE (circular otherwise). Fail LOUD on a short parse — never silently
    return a subset that would hide a class."""
    txt = open(doc, encoding="utf-8").read()
    classes = sorted(set(re.findall(r"^\|\s*(S\d+)\s*\|", txt, re.M)), key=lambda s: int(s[1:]))
    if len(classes) < 11:
        raise SystemExit(f"coverage: taxonomy parse found {classes} (<11) — ledger format changed; FIX before trusting")
    return classes


def detector_classes(d, cells):
    """Classes a detector is RESPONSIBLE for — derived ACROSS the §8 manifest bifurcation so a missing `covers`
    on an exec manifest cannot manufacture a false blind spot:
    `covers` (LLM list) ∪ `class` (exec singular) ∪ classes of conformance cells the detector ships."""
    s = set(d.get("covers", []))
    if d.get("class"):
        s.add(d["class"])
    s |= {c["class"] for c in cells if c.get("_dir", "").startswith(d["_dir"] + os.sep)}
    return s


def solution_classes(s):
    """Classes a SolutionAdapter manifest covers: `class` (singular) ∪ `applies_to.class`. Mirror of
    detector_classes for the solution side (kept narrow — solutions declare one target class)."""
    out = set()
    if s.get("class"):
        out.add(s["class"])
    ap = s.get("applies_to") or {}
    if ap.get("class"):
        out.add(ap["class"])
    return out


def resolution_grade(cls, solutions):
    """Resolution lifecycle grade for a class: 'auto' (a syntactic-local adapter covers it) | 'suggested' (a
    located-suggestion adapter) | 'none'. GRADED, never a gap — a class with no SolutionAdapter is surfaced
    'none', never a CI failure (resolution gated HARDER: absence is honest, not blocking)."""
    rungs = {s.get("rung") for s in solutions if cls in solution_classes(s)}
    if "syntactic-local" in rungs:
        return "auto"
    if "located-suggestion" in rungs:
        return "suggested"
    return "none"


def precision_grade(cls, cells_by_cls, prec_by_cell, ships_safe_fn):
    """Anti-canary (precision / GREEN-on-safe) measurement grade for a class — the negative-polarity twin of the
    Re-discovery rate (goal-item #7). Tri-state:
      'measured' — a k>=3, hash-current anti-canary record exists for >=1 cell of the class (consumed from
                   recall_gate.run() over precision_records.json).
      'present'  — a safe/anti-canary cell ships (ships_safe) but its GREEN-on-safe rate is UNMEASURED (rate-pending).
      'none'     — no safe cell.
    Returns (grade, rate_str|None). rate_str surfaces the WORST measured anti-canary rate (min over the class's
    measured cells) so a BELOW_FLOOR cell is NOT masked behind a green 'measured' word (recall_gate's
    rate_validated INCLUDES below-floor records by design). GRADED, never a gap — the HARD STALE/k<3 teeth live in
    check.sh's recall_gate-on-precision line, NOT here (mirror of resolution_grade)."""
    measured = [prec_by_cell[c["id"]] for c in cells_by_cls.get(cls, []) if c.get("id") in prec_by_cell]
    if measured:
        worst = min(measured, key=lambda d: d["rate"])
        return "measured", f"{round(worst['rate'] * worst['k'])}/{worst['k']}"
    if ships_safe_fn(cls):
        return "present", None
    return "none", None


def _solution_conformance_gap(s, root):
    """A solution declaring a rung is an ADMISSION claim — it must ship BOTH its exec script AND a cells/
    conformance artifact, else its resolution grade is a false coverage claim (no-false-coverage; mirror of the
    detector admission where a 'deterministic' grade requires a shipped conformance cell). Existence-by-convention
    like ships_safe; pytest enforces GREEN. Returns a detail string when the admission artifact is missing, else None."""
    if not s.get("rung"):
        return None
    exec_cmd = s.get("exec") or []
    script = next((a for a in exec_cmd[1:] if not a.startswith("-")), None)
    script_abs = script if (script and os.path.isabs(script)) else (os.path.join(root, script) if script else None)
    if not (script_abs and os.path.isfile(script_abs)):
        return f"solution '{s.get('id')}' declares rung '{s.get('rung')}' but its exec script is missing"
    cdir = os.path.join(s.get("_dir", ""), "cells")
    if not (os.path.isdir(cdir) and any(n.endswith(".json") for n in os.listdir(cdir))):
        return f"solution '{s.get('id')}' declares rung '{s.get('rung')}' but ships no cells/ conformance artifact"
    return None


def load_confirmed():
    try:
        return json.load(open(CONFIRMED, encoding="utf-8"))
    except (FileNotFoundError, OSError, ValueError):
        return []


def _repo_of(cell):
    """dedup-key repo component (§5). Provenance lands with fable (§6); today cells are single-repo, so fall back
    to an explicit field, else "" — fix_sha+symbol+class already disambiguate within one repo."""
    prov = cell.get("provenance")
    if isinstance(prov, dict) and prov.get("repo"):
        return prov["repo"]
    return cell.get("repo", "")


def build(cells, detectors, confirmed, classes, solutions=(), precision_validated=()):
    det_cov = {d["id"]: detector_classes(d, cells) for d in detectors}
    det_kind = {d["id"]: d.get("kind", "exec" if (d.get("class") or any(c.get("_dir", "").startswith(d["_dir"] + os.sep) for c in cells)) else "llm") for d in detectors}
    cls_dets = {cls: sorted(i for i, cov in det_cov.items() if cls in cov) for cls in classes}
    cells_by_cls = {}
    for c in cells:
        cells_by_cls.setdefault(c["class"], []).append(c)

    def ships_safe(cls):
        for d in detectors:
            cdir = os.path.join(d["_dir"], "cells")
            if cls in det_cov[d["id"]] and os.path.isdir(cdir) and any("safe" in n.lower() for n in os.listdir(cdir)):
                return True
        return False

    prec_by_cell = {d["cell_id"]: d for d in precision_validated}  # cell_id -> {rate,k,...} from recall_gate

    # PROJECTION first: one row per defect, deduped on class+repo+symbol+fix_sha (§5). Rate-capability is then
    # graded on DEFECTS, not raw cells — a cross-file defect is N cells but ONE ground-truth sample (e.g. S1's
    # 2FA pair shares fix ead618d), so counting cells would falsely promote a thin class to "rate".
    seen, rows = {}, []
    for c in cells:
        key = (c.get("class"), _repo_of(c), c.get("canonical_symbol", ""), c.get("fix_sha", ""))
        if key in seen:
            continue
        seen[key] = True
        cls = c.get("class")
        ratcheted = any(cf.get("class") == cls and (cf.get("file") == c.get("file") or
                        (cf.get("symbol", "") and cf.get("symbol", "") in c.get("canonical_symbol", "")))
                        for cf in confirmed)
        rows.append({
            "defect": c.get("id"), "class": cls, "file": c.get("file"), "fix_sha": c.get("fix_sha"),
            "detectors": cls_dets.get(cls, []), "ratcheted": ratcheted,
            # resolution = class-level "how to fix" (§5), DERIVED from solution.json manifests (auto|suggested|none).
            "resolution": resolution_grade(cls, solutions),
            "provenance": c.get("provenance") or {},   # Discovered record (plain data); {} when absent
        })

    defects_by_cls = {}
    for r in rows:
        defects_by_cls.setdefault(r["class"], 0)
        defects_by_cls[r["class"]] += 1

    grades = []
    for cls in classes:
        dets = cls_dets[cls]
        n_def = defects_by_cls.get(cls, 0)
        # deterministic ONLY if an exec detector ships a conformance cell OF THIS CLASS — NOT merely covers it.
        # (a wrapped Semgrep covers:[S4,S5,S6] shipping only an S4 cell must NOT grade S5 'deterministic'.)
        deterministic = any(
            det_kind[d["id"]] != "llm"
            and any(c.get("class") == cls and c.get("_dir", "").startswith(d["_dir"] + os.sep) for c in cells)
            for d in detectors if d["id"] in dets)
        disc_grade, disc_rate = precision_grade(cls, cells_by_cls, prec_by_cell, ships_safe)
        grades.append({
            "class": cls, "n_cells": len(cells_by_cls.get(cls, [])), "n_defects": n_def,
            # rate-CAPABLE (>=3 deduped defects = enough ground truth to roll a RATE) — NOT a claim recall WAS
            # rolled k>=3; that measurement is bench's, never derivable here. point-estimate = 1-2 defects.
            "sample": "rate-capable" if n_def >= 3 else "point-estimate" if n_def >= 1 else "none",
            "detectors": dets,
            "coverage": "deterministic" if deterministic else "asserted" if dets else "none",
            "discriminator_tested": disc_grade,   # tri-state grade (was the ships_safe bool)
            "discriminator_rate": disc_rate,       # "3/3" | None — surfaced so a below-floor cell is not masked
        })

    gaps = []
    for cls in classes:
        if not cells_by_cls.get(cls):
            gaps.append({"kind": "blind-spot", "class": cls, "detail": "no cell — class has zero ground truth"})
        if not cls_dets[cls]:
            gaps.append({"kind": "uncovered", "class": cls, "detail": "no responsible detector"})
    for c in cells:
        if not cls_dets.get(c.get("class")):
            gaps.append({"kind": "cell-no-detector", "class": c.get("class"), "detail": c.get("id")})
    for cf in confirmed:
        if not cls_dets.get(cf.get("class")):
            gaps.append({"kind": "inert-ratchet", "class": cf.get("class"),
                         "detail": f"ratchet entry {cf.get('ref', cf.get('file'))} has no covering detector"})
    for s in solutions:
        g = _solution_conformance_gap(s, ROOT)
        if g:
            gaps.append({"kind": "solution-no-conformance",
                         "class": next(iter(solution_classes(s)), "?"), "detail": g})

    summary = {
        "n_classes": len(classes), "n_cells": len(cells), "n_defects": len(rows),
        "rate_capable": [g["class"] for g in grades if g["sample"] == "rate-capable"],
        "point_estimate": [g["class"] for g in grades if g["sample"] == "point-estimate"],
        "deterministic": [g["class"] for g in grades if g["coverage"] == "deterministic"],
        "discriminator_mutant": "unbuilt — 0 artifacts (rate-pending BLIND_SPOT); this gate does NOT claim "
                                "mutant coverage (rename/cosmetic/polarity/null anti-overfit mutation)",
        "blind_spots": [g["class"] for g in grades if g["n_cells"] == 0],
        "ok": not gaps,
        "note": "ok=True = every class has a cell + a responsible detector + a live ratchet. It does NOT mean "
                "11/11 recall-validated at rate — `sample` grades GROUND-TRUTH DEPTH (rate-capable = >=3 deduped "
                "defects), NOT whether recall was rolled k>=3 (that is bench's measurement, not derivable here).",
    }
    return {"summary": summary, "classes": grades, "defects": rows, "gaps": gaps}


def _md(reg):
    s = reg["summary"]
    out = [f"# Coverage registry — {s['n_defects']} defects / {s['n_classes']} classes / ok={s['ok']}", "",
           "| class | n | sample | coverage | discrim | detectors |", "|---|---|---|---|---|---|"]
    for g in reg["classes"]:
        disc = g["discriminator_tested"] + (f" {g['discriminator_rate']}" if g.get("discriminator_rate") else "")
        out.append(f"| {g['class']} | {g['n_cells']} | {g['sample']} | {g['coverage']} | "
                   f"{disc} | {', '.join(g['detectors']) or '—'} |")
    out += ["", f"_discriminator-mutant axis: {s['discriminator_mutant']}_"]
    if reg["gaps"]:
        out += ["", "## GAPS (CI-fail)"] + [f"- [{g['kind']}] {g['class']}: {g['detail']}" for g in reg["gaps"]]
    out += ["", "## Lifecycle (one row per defect)",
            "| defect | class | discovered | re-discovery | resolution | prevention |",
            "|---|---|---|---|---|---|"]
    for r in reg["defects"]:
        prov = r.get("provenance") or {}
        discovered = prov.get("audit") or prov.get("discovered_by") or r.get("fix_sha") or "—"
        redet = ", ".join(r["detectors"]) or "—"
        prevention = "ratchet" if r["ratcheted"] else "none"
        out.append(f"| {r['defect']} | {r['class']} | {discovered} | {redet} | {r['resolution']} | {prevention} |")
    return "\n".join(out)


def main():
    ap = argparse.ArgumentParser(description="derived coverage registry / blind-spot oracle (ARCHITECTURE §5)")
    ap.add_argument("--check", action="store_true", help="CI gate: exit 1 on any structural gap")
    ap.add_argument("--md", action="store_true", help="render the one-row-per-defect projection as markdown")
    a = ap.parse_args()
    cells = bench.load_cells()
    detectors, skipped = manifest.load(ROOT)
    solutions, sol_skipped = manifest.load(ROOT, kind="solution")
    _, prec_res = recall.run(PRECISION_RECORDS, recall.DEF_CORPUS, recall.DEF_DETECTORS, recall.DEF_FLOOR, False)
    precision_validated = prec_res.get("rate_validated", [])  # .get guards the missing-file shape (no key there)
    reg = build(cells, detectors, load_confirmed(), taxonomy_classes(), solutions, precision_validated)
    if skipped:  # a malformed manifest is reduced coverage, never a silent pass (no-false-clean)
        reg["summary"]["skipped_manifests"] = [p for p, _ in skipped]
        reg["gaps"].append({"kind": "manifest-unparsable", "class": "?",
                            "detail": f"{len(skipped)} detector.json failed to parse — coverage UNRELIABLE"})
        reg["summary"]["ok"] = False
    if sol_skipped:  # a malformed solution.json under-grades resolution — surface it, but resolution is GRADED not gated
        reg["summary"]["skipped_solutions"] = [p for p, _ in sol_skipped]
    print(_md(reg) if a.md else json.dumps(reg, indent=2))
    if a.check and reg["gaps"]:
        print(f"\ncoverage: {len(reg['gaps'])} structural gap(s) — NOT a coverage clean", file=sys.stderr)
        sys.exit(1)
    sys.exit(0)


if __name__ == "__main__":
    main()