From a36956b1e1580d28259d0bdccbe96f2ecd5922a9 Mon Sep 17 00:00:00 2001 From: Croissant Le Doux Date: Mon, 3 Aug 2026 19:52:32 -0400 Subject: [PATCH] corpus: align determinability key with the schema column MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The truth files used `determinableFromResponseDocument`; the schema column and gateEvaluableFromResponse() use `determinableFromResponse`. A silent name drift means the flag reads as its default (true), and R-B6C — "two bid copies submitted", which no response document can ever establish — starts failing bidders on a packaging detail. Renamed in rfp.truth.json and validate-responses.py, and added the mirror check to validate-truth.py so the drift cannot recur unnoticed: gates 13, evaluable from the response 12, excluded 1 excluded: R-B6C Two bid copies submitted. Co-Authored-By: Claude Opus 5 (1M context) --- corpus/friendship-pcs/rfp.truth.json | 2 +- corpus/validate-responses.py | 2 +- corpus/validate-truth.py | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/corpus/friendship-pcs/rfp.truth.json b/corpus/friendship-pcs/rfp.truth.json index 39b3e27..0d68edc 100644 --- a/corpus/friendship-pcs/rfp.truth.json +++ b/corpus/friendship-pcs/rfp.truth.json @@ -241,7 +241,7 @@ "page": 5, "text": "Two bid copies submitted.", "criterionIds": [], - "determinableFromResponseDocument": false, + "determinableFromResponse": false, "$comment": "A physical submission fact, not a document-content fact. No extraction of any response can establish it, and no better parsing would help. Kept as a mandatory gate because the SFA does enforce it — but it must be satisfied out of band. The schema has no concept for this today; see the note raised on #10." }, { diff --git a/corpus/validate-responses.py b/corpus/validate-responses.py index ef9eb7c..9943e65 100644 --- a/corpus/validate-responses.py +++ b/corpus/validate-responses.py @@ -13,7 +13,7 @@ gates = {r['id'] for r in rfp['requirements'] if r['kind'] == 'mandatory_gate'} # A gate whose satisfaction is invisible to the response document cannot be scored # from it. Excluded from the silent-gate check; see requirementNotes in the RFP truth. undeterminable = {r['id'] for r in rfp['requirements'] - if r.get('determinableFromResponseDocument') is False} + if r.get('determinableFromResponse') is False} COVERAGE = {'answered', 'not_answered', 'indeterminate'} errs, warns = [], [] diff --git a/corpus/validate-truth.py b/corpus/validate-truth.py index 409ea70..dcc0c79 100644 --- a/corpus/validate-truth.py +++ b/corpus/validate-truth.py @@ -77,6 +77,22 @@ for cid in sec: if cid not in crit: errs.append(f"bindingMechanism section -> unknown criterion {cid}") gates = [r for r in t['requirements'] if r['kind'] == 'mandatory_gate'] + +# Mirrors gateEvaluableFromResponse() in src/scoring/routing.ts. The key read +# here is the SCHEMA COLUMN NAME on purpose — the two names drifted once, and a +# silent drift means the flag reads as its default and a bidder is disqualified +# over something no document could have proven. +evaluable = [r for r in gates if r.get('determinableFromResponse', True)] +excluded = [r for r in gates if not r.get('determinableFromResponse', True)] +print(f" ok gates {len(gates)}, evaluable from the response {len(evaluable)}, " + f"excluded {len(excluded)}") +for r in excluded: + print(f" excluded: {r['id']:14} {r['text'][:50]}") +for r in t['requirements']: + if r.get('determinableFromResponse') is False and r['kind'] != 'mandatory_gate': + warns.append(f"{r['id']} is not determinable but is not a gate — the flag " + f"only changes gate evaluation, so this has no effect") + kinds = {} for r in t['requirements']: kinds[r['kind']] = kinds.get(r['kind'], 0) + 1 print(f"\n requirements {len(t['requirements'])} criteria {len(t['criteria'])} knownMisses {len(t['knownMisses'])}")