19e7ffe92a
Covers all nine stress cases from #25, #15 and #16. The lowest bid is disqualified on two missing health documents and the highest bid is strongest on the SFA's own geographic-preference criterion, so a ranking that cannot represent absence gets the field exactly backwards. Deliberate traps: a planted $400 arithmetic error in a stated total, an unverifiable total whose unit price lives only in the external Excel, an unlabelled exception, a non-priceable exception that is really a conditional withdrawal, and an answered-but-negative response. Second schema finding, surfaced by the validator: R-B6C is a mandatory gate that no response document can establish. Coverage over it is unknowable rather than absent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
95 lines
4.8 KiB
Python
95 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Cross-check response truth files against their RFP truth file.
|
|
|
|
Run: python3 corpus/validate-responses.py corpus/friendship-pcs
|
|
Hand-authored ground truth — nothing else checks these.
|
|
"""
|
|
import json, sys, glob, os
|
|
|
|
root = sys.argv[1] if len(sys.argv) > 1 else 'friendship-pcs'
|
|
rfp = json.load(open(os.path.join(root, 'rfp.truth.json')))
|
|
reqs = {r['id']: r for r in rfp['requirements']}
|
|
gates = {r['id'] for r in rfp['requirements'] if r['kind'] == 'mandatory_gate'}
|
|
# A gate whose satisfaction is invisible to the response document cannot be scored
|
|
# from it. Excluded from the silent-gate check; see requirementNotes in the RFP truth.
|
|
undeterminable = {r['id'] for r in rfp['requirements']
|
|
if r.get('determinableFromResponseDocument') is False}
|
|
COVERAGE = {'answered', 'not_answered', 'indeterminate'}
|
|
|
|
errs, warns = [], []
|
|
print(f"RFP: {len(reqs)} requirements, {len(gates)} gates\n")
|
|
|
|
for path in sorted(glob.glob(os.path.join(root, 'responses', '*.truth.json'))):
|
|
t = json.load(open(path))
|
|
v = t['vendor']['id']
|
|
answered = {}
|
|
for a in t['answers']:
|
|
rid = a['requirementId']
|
|
if rid not in reqs:
|
|
errs.append(f"{v}: answer -> unknown requirement {rid}")
|
|
continue
|
|
if a['coverage'] not in COVERAGE:
|
|
errs.append(f"{v}: {rid} bad coverage '{a['coverage']}'")
|
|
if rid in answered:
|
|
errs.append(f"{v}: duplicate answer for {rid}")
|
|
answered[rid] = a
|
|
|
|
missing = sorted(set(reqs) - set(answered))
|
|
if missing:
|
|
warns.append(f"{v}: {len(missing)} requirements have no answer entry: {', '.join(missing[:4])}{'...' if len(missing) > 4 else ''}")
|
|
|
|
# gate failures must agree with the declared outcome
|
|
declared = set(t['expectedOutcome'].get('failedGates', []))
|
|
actual = {r for r, a in answered.items()
|
|
if r in gates and (a.get('isGateFailure') or a['coverage'] == 'not_answered')}
|
|
flagged = {r for r, a in answered.items() if a.get('isGateFailure')}
|
|
if not declared <= flagged:
|
|
errs.append(f"{v}: declared failedGates {sorted(declared - flagged)} not marked isGateFailure on the answer")
|
|
if t['expectedOutcome']['gatesPassed'] and declared:
|
|
errs.append(f"{v}: gatesPassed=true but failedGates is non-empty")
|
|
if not t['expectedOutcome']['gatesPassed'] and not declared:
|
|
errs.append(f"{v}: gatesPassed=false but no failedGates declared")
|
|
|
|
# unflagged gate misses are a real signal, not necessarily an error
|
|
silent = actual - declared - set(t['expectedOutcome'].get('atRiskGates', [])) - undeterminable
|
|
if silent:
|
|
warns.append(f"{v}: gate(s) not_answered but not declared failed or at-risk: {sorted(silent)}")
|
|
|
|
for e in t.get('exceptions', []):
|
|
rid = e.get('requirementId')
|
|
if rid is not None and rid not in reqs:
|
|
errs.append(f"{v}: exception {e['id']} -> unknown requirement {rid}")
|
|
|
|
counts = {}
|
|
for a in answered.values():
|
|
counts[a['coverage']] = counts.get(a['coverage'], 0) + 1
|
|
exc = len(t.get('exceptions', []))
|
|
labelled = sum(1 for e in t.get('exceptions', []) if e.get('labelled'))
|
|
print(f" {v:11} binding={t['bindingMechanism']['level']:34} "
|
|
f"answered={counts.get('answered',0):2} indet={counts.get('indeterminate',0)} "
|
|
f"absent={counts.get('not_answered',0)} exceptions={exc} ({labelled} labelled) "
|
|
f"gates={'PASS' if t['expectedOutcome']['gatesPassed'] else 'FAIL'}")
|
|
|
|
# corpus-level coverage of the stress-case checklist
|
|
files = [json.load(open(p)) for p in glob.glob(os.path.join(root, 'responses', '*.truth.json'))]
|
|
checks = {
|
|
'tabbed compliance matrix': any(f['bindingMechanism']['level'] == 'tabbed_restatement' for f in files),
|
|
'free prose, no anchors': any(f['bindingMechanism']['level'] == 'free_prose' for f in files),
|
|
'rate function, not scalar': len({f['priceQuote']['rateStructure']['kind'] for f in files}) >= 3,
|
|
'partially non-responsive': any(not f['expectedOutcome']['gatesPassed'] for f in files),
|
|
'exceptions with price deltas': any(e.get('priceDeltaIfDeclined') for f in files for e in f.get('exceptions', [])),
|
|
'unlabelled exception': any(e.get('labelled') is False for f in files for e in f.get('exceptions', [])),
|
|
'labelled known-misses in RFP': len(rfp.get('knownMisses', [])) >= 3,
|
|
'miskinded-as-informational': any(k['type'] == 'miskinded_as_informational' for k in rfp['knownMisses']),
|
|
'answered-but-negative': any('negative' in str(a.get('$comment','')).lower() for f in files for a in f['answers']),
|
|
}
|
|
print("\n stress-case checklist")
|
|
for k, ok in checks.items():
|
|
print(f" {'x' if ok else ' '} {k}")
|
|
if not ok: warns.append(f"stress case not covered: {k}")
|
|
|
|
for w in warns: print(f"\n WARN {w}")
|
|
for e in errs: print(f"\n ERR {e}")
|
|
print("\nVALID" if not errs else "\nINVALID")
|
|
sys.exit(1 if errs else 0)
|