#!/usr/bin/env python3 """Validate a corpus truth file's internal consistency. Run: python3 corpus/validate-truth.py corpus//rfp.truth.json These files are hand-authored ground truth, so nothing else checks them. """ import json, sys path = sys.argv[1] if len(sys.argv) > 1 else 'friendship-pcs/rfp.truth.json' t = json.load(open(path)) errs, warns = [], [] crit = {c['id']: c for c in t['criteria']} reqs = {r['id']: r for r in t['requirements']} kms = {k['id']: k for k in t['knownMisses']} nest = t['criteriaNotes']['nestingIsInconsistent'] for r in t['requirements']: for cid in r.get('criterionIds', []): if cid not in crit: errs.append(f"{r['id']} -> unknown criterion {cid}") if 'knownMissId' in r and r['knownMissId'] not in kms: errs.append(f"{r['id']} -> unknown knownMiss {r['knownMissId']}") for k in t['knownMisses']: if k['requirementId'] not in reqs: errs.append(f"{k['id']} -> unknown requirement {k['requirementId']}") for c in t['criteria']: if c['parentId'] and c['parentId'] not in crit: errs.append(f"{c['id']} -> unknown parent {c['parentId']}") for pid in nest['weightedParents']: kids = [c for c in t['criteria'] if c['parentId'] == pid] s = sum(c['maxPoints'] for c in kids) if s != crit[pid]['maxPoints']: errs.append(f"{pid}: children sum {s} != parent {crit[pid]['maxPoints']}") else: print(f" ok {pid:14} weighted children sum {s} == parent") for pid in nest['guidanceOnlyParents']: kids = [c for c in t['criteria'] if c['parentId'] == pid] if kids: errs.append(f"{pid} declared guidance-only but has {len(kids)} child criteria") else: print(f" ok {pid:14} guidance-only, no child criteria") top = sum(c['maxPoints'] for c in t['criteria'] if c['parentId'] is None) tot = t['criteriaNotes']['totalPoints'] print(f" {'ok ' if top == tot else 'ERR '} top-level points sum {top} vs declared {tot}") if top != tot: errs.append(f"top-level sum {top} != {tot}") pts = sorted(a['points'] for a in t['ratingScale']['anchors']) if pts != list(range(t['ratingScale']['min'], t['ratingScale']['max'] + 1)): errs.append(f"rating scale not contiguous: {pts}") else: print(f" ok rating scale {pts[0]}-{pts[-1]}, {len(pts)} anchors") # The scoring-level rule, mirrored from src/scoring/routing.ts: a score commits # at any criterion with NO WEIGHTED CHILDREN. Printed so the corpus and the code # cannot silently disagree about which criteria a committee actually scores. scoreable = [c for c in t['criteria'] if not any(k['parentId'] == c['id'] and (k.get('maxPoints') or 0) > 0 for k in t['criteria'])] sc_total = sum(c['maxPoints'] for c in scoreable) print(f" ok scoreable criteria {len(scoreable)} of {len(t['criteria'])}, " f"points sum {sc_total}") if sc_total != tot: errs.append(f"scoreable points sum {sc_total} != declared total {tot}") print(" " + ", ".join(f"{c['id']}({c['maxPoints']})" for c in scoreable)) mapped = {c for r in t['requirements'] for c in r.get('criterionIds', [])} for cid, c in crit.items(): kids = [x for x in t['criteria'] if x['parentId'] == cid] if cid not in mapped and not kids: warns.append(f"criterion {cid} ({c['label'][:38]}) has no requirement mapped — unscoreable") sec = {s['mapsToCriterion'] for s in t['bindingMechanism']['requiredSections']} - {None} for cid in sec: if cid not in crit: errs.append(f"bindingMechanism section -> unknown criterion {cid}") gates = [r for r in t['requirements'] if r['kind'] == 'mandatory_gate'] kinds = {} for r in t['requirements']: kinds[r['kind']] = kinds.get(r['kind'], 0) + 1 print(f"\n requirements {len(t['requirements'])} criteria {len(t['criteria'])} knownMisses {len(t['knownMisses'])}") print(f" by kind: " + ", ".join(f"{k}={v}" for k, v in sorted(kinds.items()))) for w in warns: print(f" WARN {w}") for e in errs: print(f" ERR {e}") print("\nVALID" if not errs else "\nINVALID") sys.exit(1 if errs else 0)