#!/usr/bin/env python3
"""Recompute the held-out headline numbers from the frozen inputs and check them
against HOLDOUT_COMPARISON_RESULTS.json. No network. Run: python3 verify_holdout.py

Inputs (all in this directory):
  sealed/baseline_normalized.json         fixed-surface baseline, R_B-derived tier
  sealed/guard_directed_normalized.json   guard-directed resolver
  HOLDOUT_GROUND_TRUTH.final.json         final ground truth (121 rows)
  HOLDOUT_COMPARISON_RESULTS.json         the scored result reported in the paper
  R_B_adapter.py                          frozen baseline tier adapter

The exact-match metrics (tier, paired McNemar, AUTHORITY_OPAQUE, coverage,
non-upgradeable, abstention) are recomputed here and asserted. Dependency-category
and terminal-type committed-correctness are the soft metrics: they depend on the
documented text-to-category mapping in HOLDOUT_SCORING_PROTOCOL.md, so this script
reports the stored values for them rather than re-deriving the mapping.
"""
import json, os, sys

P = os.path.dirname(os.path.abspath(__file__))
def load(f): return json.load(open(os.path.join(P, f)))

gt = {(r['protocol_slug'], r['chain']): r for r in load('HOLDOUT_GROUND_TRUTH.final.json')['ground_truth']}
B = {(r['protocol_slug'], r['chain']): r for r in load('sealed/baseline_normalized.json')['predictions']}
G = {(r['protocol_slug'], r['chain']): r for r in load('sealed/guard_directed_normalized.json')['predictions']}
stored = load('HOLDOUT_COMPARISON_RESULTS.json')['full_121']

T123 = {'T1', 'T2', 'T3'}
keys = list(gt)
upg = [k for k in keys if gt[k]['upgrade_path_tier'] != 'NON_UPGRADEABLE']
tier_elig = [k for k in keys if gt[k]['upgrade_path_tier'] in T123]
nonupg = [k for k in keys if gt[k]['upgrade_path_tier'] == 'NON_UPGRADEABLE']
gt_opaque = [k for k in keys if gt[k]['upgrade_path_tier'] == 'AUTHORITY_OPAQUE']

def ptier(P, k): return P[k].get('predicted_upgrade_path_tier')
def committed_tier(P, k): return ptier(P, k) in T123
def tier_correct(P, k): return committed_tier(P, k) and ptier(P, k) == gt[k]['upgrade_path_tier']
def is_opaque(P, k): return bool(P[k].get('authority_opaque')) or P[k].get('status') == 'AUTHORITY_OPAQUE'

checks = []
def chk(name, got, exp): checks.append((name, got, exp, got == exp))

# --- tier over GT T1/T2/T3 = 59 ---
for label, Pd in (('baseline', B), ('guard', G)):
    committed = sum(committed_tier(Pd, k) for k in tier_elig)
    correct = sum(tier_correct(Pd, k) for k in tier_elig)
    s = stored['tier'][label]
    chk(f'tier {label} eligible', len(tier_elig), s['eligible'])
    chk(f'tier {label} committed', committed, s['committed'])
    chk(f'tier {label} correct', correct, s['correct'])

# --- paired McNemar over the 59 ---
bo = sum(tier_correct(B, k) and not tier_correct(G, k) for k in tier_elig)
go = sum(tier_correct(G, k) and not tier_correct(B, k) for k in tier_elig)
bb = sum(tier_correct(B, k) and tier_correct(G, k) for k in tier_elig)
nn = sum((not tier_correct(B, k)) and (not tier_correct(G, k)) for k in tier_elig)
chi2 = round((abs(bo - go) - 1) ** 2 / (bo + go), 3)
sp = stored['paired_tier_over_T1T2T3']
chk('paired both_correct', bb, sp['both_correct'])
chk('paired baseline_only', bo, sp['baseline_only'])
chk('paired guard_only', go, sp['guard_only'])
chk('paired both_incorrect', nn, sp['both_incorrect'])
chk('paired mcnemar_chi2_cc', chi2, sp['mcnemar_chi2_cc'])

# --- AUTHORITY_OPAQUE (gt = 8) ---
for label, Pd in (('baseline', B), ('guard', G)):
    sysop = [k for k in keys if is_opaque(Pd, k)]
    tp = [k for k in sysop if k in gt_opaque]
    s = stored['opaque'][label]
    chk(f'opaque {label} TP', len(tp), s['TP'])
    chk(f'opaque {label} FN', len(gt_opaque) - len(tp), s['FN'])
    chk(f'opaque {label} FP', len(sysop) - len(tp), s['FP'])
    chk(f'opaque {label} precision denom', len(sysop), s['precision'][1])

# --- dependency COVERAGE (committed = dependency resolved) ---
for label, Pd in (('baseline', B), ('guard', G)):
    dep_committed = sum(Pd[k].get('dependency_status') == 'RESOLVED' for k in upg)
    chk(f'dependency {label} committed', dep_committed, stored['dependency'][label]['committed'])

# --- non-upgradeable false-upgradeable (predicted a live tier on an immutable core) ---
for label, Pd in (('baseline', B), ('guard', G)):
    false_upg = sum(committed_tier(Pd, k) for k in nonupg)
    chk(f'non-upgradeable {label} incorrectly_upgradeable', false_upg,
        stored['non_upgradeable'][label]['incorrectly_upgradeable'])

# --- abstention: committed-tier count over all 121 ---
for label, Pd in (('baseline', B), ('guard', G)):
    chk(f'abstention {label} committed_tier', sum(committed_tier(Pd, k) for k in keys),
        stored['abstention'][label]['committed_tier'])

# --- R_B reproduces the sealed baseline_normalized.json from baseline_raw.json ---
try:
    import R_B_adapter
    ok_rb, n_rb = R_B_adapter.reproduce_check(P)
    chk('R_B reproduces sealed baseline_normalized', ok_rb, True)
except Exception as e:  # pragma: no cover
    chk('R_B reproduces sealed baseline_normalized', f'ERROR: {e}', True)

# --- report ---
print(f"{'check':44}{'got':>10}{'exp':>10}  status")
ok = 0
for name, got, exp, passed in checks:
    print(f"{name:44}{str(got):>10}{str(exp):>10}  {'MATCH' if passed else 'MISMATCH'}")
    ok += passed
print(f"\n{ok}/{len(checks)} held-out figures reconcile")

# Soft metrics reported from HOLDOUT_COMPARISON_RESULTS.json for reference. Dependency
# category-correctness depends on the documented text-to-category mapping, and terminal
# is scored at the type level; both follow HOLDOUT_SCORING_PROTOCOL.md rather than being
# re-derived here.
print("\nSoft metrics (per HOLDOUT_SCORING_PROTOCOL.md, reported not recomputed):")
for label in ('baseline', 'guard'):
    d = stored['dependency'][label]['committed_correct']
    tc = stored['terminal'][label]['coverage']
    t = stored['terminal'][label]['committed_correct']
    print(f"  {label:8} dependency committed-correct {d[0]}/{d[1]}   "
          f"terminal coverage {tc[0]}/{tc[1]}   terminal committed-correct {t[0]}/{t[1]}")

if ok != len(checks):
    print("MISMATCHES ABOVE"); sys.exit(1)