From 19e7ffe92a9cbe4f122d0aebda120ebfbd83b9be Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 3 Aug 2026 18:39:08 -0400 Subject: [PATCH] Add three synthesized Friendship PCS responses with ground truth Covers all nine stress cases from #25, #15 and #16. The lowest bid is disqualified on two missing health documents and the highest bid is strongest on the SFA's own geographic-preference criterion, so a ranking that cannot represent absence gets the field exactly backwards. Deliberate traps: a planted $400 arithmetic error in a stated total, an unverifiable total whose unit price lives only in the external Excel, an unlabelled exception, a non-priceable exception that is really a conditional withdrawal, and an answered-but-negative response. Second schema finding, surfaced by the validator: R-B6C is a mandatory gate that no response document can establish. Coverage over it is unknowable rather than absent. Co-Authored-By: Claude Opus 5 (1M context) --- corpus/friendship-pcs/NOTES.md | 59 ++ .../anacostia-farm-collective.truth.json | 102 ++++ .../chesapeake-food-service.truth.json | 237 ++++++++ .../responses/potomac-provisions.truth.json | 90 +++ .../source/anacostia-farm-collective.md | 184 ++++++ .../source/chesapeake-food-service.md | 118 ++++ .../responses/source/potomac-provisions.md | 170 ++++++ corpus/friendship-pcs/rfp.truth.json | 526 +++++++++++++++--- corpus/validate-responses.py | 94 ++++ 9 files changed, 1509 insertions(+), 71 deletions(-) create mode 100644 corpus/friendship-pcs/responses/anacostia-farm-collective.truth.json create mode 100644 corpus/friendship-pcs/responses/chesapeake-food-service.truth.json create mode 100644 corpus/friendship-pcs/responses/potomac-provisions.truth.json create mode 100644 corpus/friendship-pcs/responses/source/anacostia-farm-collective.md create mode 100644 corpus/friendship-pcs/responses/source/chesapeake-food-service.md create mode 100644 corpus/friendship-pcs/responses/source/potomac-provisions.md create mode 100644 corpus/validate-responses.py diff --git a/corpus/friendship-pcs/NOTES.md b/corpus/friendship-pcs/NOTES.md index 23a76c4..d24cb60 100644 --- a/corpus/friendship-pcs/NOTES.md +++ b/corpus/friendship-pcs/NOTES.md @@ -63,3 +63,62 @@ present, its content is not, and no amount of better PDF parsing recovers it. 149 × "shall", 64 × "must" across 52 pages. An earlier ad-hoc extraction reported zero of each; that was the extractor failing, not the document. + +--- + +# Response set + +Three synthesized responses in `responses/source/`, each paired with a truth +file. Designed so the field has a real outcome rather than three +interchangeable bids. + +| Vendor | Binding | Price | Gates | Distinguishing feature | +|---|---|---|---|---| +| Potomac Provisions | tabbed compliance matrix | $589,970 | PASS | Complete; volume-banded pricing; 18% local | +| Chesapeake Food Service | free prose, no anchors | $541,280 | **FAIL** | Cheapest and disqualified; unlabelled exception | +| Anacostia Farm Collective | named sections + enumeration | $624,315 | PASS | 61% local; three labelled exceptions; seasonal pricing | + +The lowest bid is disqualified and the highest bid is strongest on the +criterion the SFA wrote into its own rubric. That is deliberate: a ranking +that cannot represent absence puts Chesapeake first. + +## Deliberate traps + +**Planted arithmetic error (Potomac).** The stated base-year total is +$589,970; band price times scenario volume is $589,570. A $400 gap. #15 +decision 2 puts all price arithmetic in code precisely so a stated total is +never trusted over a computed one — this is the case that catches an +extractor lifting the vendor's own figure. + +**Unverifiable total (Chesapeake).** The case price exists only in the +external Excel sheet, so no computed total can be produced at all. Contrast +Potomac, where the arithmetic can be checked and fails. + +**Unlabelled exception (Chesapeake).** A price-relief request phrased as +politeness inside the Pricing section, with no heading and no delta. +Anacostia's three exceptions are explicitly headed with deltas and act as +the control. + +**Non-priceable exception (Anacostia, EX-3).** If declined, the vendor +cannot bid produce at all — a conditional withdrawal expressed as an +exception. A price-delta field cannot hold it. + +**Answered-but-negative (Anacostia).** "We are not currently an approved +USDA Foods processor." Squarely answers the requirement; the substance is a +disclosure of inability. Coverage and quality are orthogonal, and a system +that scores this like Potomac's 340,000 lbs is broken. + +**Three certifications in one sentence (Chesapeake).** Attachments B, C and +D are satisfied by a single clause naming all three. A per-requirement +extractor must split one span three ways. + +## Finding: a gate that no response document can satisfy + +`R-B6C` — "Two bid copies submitted" — is a physical submission fact. +Potomac happens to assert it; the other two do not, and no better extraction +would change that. Coverage over it is neither answered nor not_answered but +**unknowable from the artifact**. + +Surfaced by `validate-responses.py`, not by design. Treating it as +not_answered would disqualify two of three bidders over a packaging detail. +Raised against #10. diff --git a/corpus/friendship-pcs/responses/anacostia-farm-collective.truth.json b/corpus/friendship-pcs/responses/anacostia-farm-collective.truth.json new file mode 100644 index 0000000..37d9f3b --- /dev/null +++ b/corpus/friendship-pcs/responses/anacostia-farm-collective.truth.json @@ -0,0 +1,102 @@ +{ + "$comment": "Ground truth for source/anacostia-farm-collective.md. Mid-gradient binding, and the exceptions case.", + "vendor": { "id": "ANACOSTIA", "name": "Anacostia Farm Collective", "location": "Washington, DC (Ward 8)" }, + + "bindingMechanism": { + "level": "named_section_with_partial_enumeration", + "$comment": "#22's middle tier. Four required sections are correctly titled and an enclosures paragraph enumerates artifacts with RFP references inline — but there is no matrix, so gates bind by name while narrative requirements bind only by section." + }, + + "expectedOutcome": { + "gatesPassed": true, + "narrative": "All gates satisfied. Highest price at $624,315, or $652,465 if exceptions 1 and 2 are declined. Strongest on geographic preference by a wide margin (61% vs 18%), openly weakest on USDA Foods, and materially below the field on organizational scale. The right award depends entirely on how the SFA weights local content against price — which is the judgement the retailer is supposed to make and the system is not." + }, + + "answers": [ + { "requirementId": "R-A1", "coverage": "answered", "evidence": "Cover block present" }, + { "requirementId": "R-A2", "coverage": "indeterminate", "evidence": "Enclosures paragraph enumerates items in checklist order but is not the checklist form itself" }, + { "requirementId": "R-B2-LOCAL", "coverage": "answered", "evidence": "61% certified; audit right accepted; certification enclosed under B.2", "value": { "localPercent": 61 } }, + { "requirementId": "R-B6C", "coverage": "not_answered", "evidence": null }, + { "requirementId": "R-B6D", "coverage": "answered", "evidence": "DC Department of Health food establishment certificate, exp. 2022-06-30" }, + { "requirementId": "R-B7-FORMAT", "coverage": "answered", "evidence": "Four sections, titles exact" }, + { "requirementId": "R-B8-QUEST", "coverage": "answered", "evidence": "References provided in questionnaire" }, + { "requirementId": "R-B8-LICENSE", "coverage": "answered", "evidence": "DC BBL 410203000891" }, + { "requirementId": "R-B8-OCCUPANCY", "coverage": "answered", "evidence": "Enclosed" }, + { "requirementId": "R-B8-FINANCIALS", "coverage": "answered", "evidence": "FY2019 and FY2020, Mercer Group CPAs" }, + { "requirementId": "R-C1-VALUES", "coverage": "answered", "evidence": "Explicit 'Response to values statement (C.1)' subsection in Section Four" }, + { "requirementId": "R-C5-INSPECTIONS", "coverage": "answered", "evidence": "2020-09-22 and 2021-01-14, both passed" }, + { "requirementId": "R-D6-INSURANCE", "coverage": "answered", "evidence": "Certificate of insurance enclosed", + "$comment": "Enclosed but NO AMOUNTS STATED, unlike the other two bidders. The requirement is 'Insurance amounts.' A strict reading makes this indeterminate; a lenient one accepts the certificate. Deliberately left ambiguous — it is the kind of call that should reach a human." }, + { "requirementId": "R-SCHED-C", "coverage": "answered", "evidence": "Submitted on Schedule C in the Excel workbook" }, + { "requirementId": "R-ATT-A", "coverage": "answered", "evidence": "Supplies four DC charter SFAs under HSA; certification designed to support the local sourcing subsidy claim" }, + { "requirementId": "R-ATT-B", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-ATT-C", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-ATT-D", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-E1-EEO", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-E3-CLEANAIR", "coverage": "answered", "evidence": "Signed" }, + + { "requirementId": "R-USDA-FOODS", "coverage": "answered", + "evidence": "'This is our weakest area and we will not overstate it. We are not currently an approved USDA Foods processor.'", + "$comment": "ANSWERED, and answered in the NEGATIVE. The requirement is squarely addressed; the substance is a disclosure of inability. A system that treats 'answered' as 'satisfied' scores this identically to Potomac's 340,000 lbs. It must not." }, + { "requirementId": "R-C8-FARM2SCHOOL", "coverage": "answered", "evidence": "34 member farms in the Chesapeake foodshed; two within the District" }, + { "requirementId": "R-C7-QA", "coverage": "answered", "evidence": "Farm-level GAP certification, annual audit, credit at receipt" }, + { "requirementId": "R-C5-FOODSAFETY", "coverage": "answered", "evidence": "DC DoH certified facility; inspections enclosed" }, + { "requirementId": "R-C6-PEST", "coverage": "answered", "evidence": "Licensed applicator monthly, semi-annual third-party audit" }, + { "requirementId": "R-CN-LABELS", "coverage": "answered", + "evidence": "'Whole produce does not carry CN labels; for those items we supply the crediting documentation the meal pattern requires.'", + "$comment": "A QUALIFIED answer — satisfies the requirement for processed items and explains why it cannot for whole produce. Neither fully answered nor a gap." }, + { "requirementId": "R-D1-DELIVERY", "coverage": "answered", "evidence": "Addressed, and subject to Exception 1 on delivery windows", + "hasException": "EX-1" }, + { "requirementId": "R-B5-PRICING-TERMS", "coverage": "answered", "evidence": "Firm within stated bands", "hasException": "EX-2" }, + { "requirementId": "R-D8-EPA", "coverage": "answered", "evidence": "Renegotiation of the out-of-region band, per Exception 2", "hasException": "EX-2" }, + { "requirementId": "R-C4-MENUS", "coverage": "not_answered", "evidence": null }, + { "requirementId": "R-D3-RECORDS", "coverage": "not_answered", "evidence": null } + ], + + "priceQuote": { + "rateStructure": { + "kind": "seasonal_piecewise", + "$comment": "The hardest of the three, and the one most likely to break a naive model. Cost depends on WHEN volume falls, not only how much — so derived_cost cannot be computed without the school calendar. #22 found exactly this shape in the wild (SLC rent varying by contract year and sales threshold).", + "unit": "case", + "baseProduceCasePrice": 31.20, + "appliesTo": "produce only; grocery, dairy and protein are flat on Schedule C", + "bands": [ + { "id": "peak_regional", "months": [6, 7, 8, 9, 10], "multiplier": 0.88 }, + { "id": "shoulder", "months": [4, 5, 11], "multiplier": 1.00 }, + { "id": "out_of_region", "months": [12, 1, 2, 3], "multiplier": 1.19 } + ], + "requiresCalendar": true, + "$calendarNote": "The 180-day school calendar falls disproportionately outside the peak band — the vendor says so explicitly. Any cost comparison that ignores the calendar flatters this bid." + }, + "statedBaseYearTotal": 624315, + "statedTotalIfExceptionsDeclined": 652465, + "$verify": "624315 + 18400 + 9750 = 652465. Internally consistent. Contrast Potomac's planted $400 error." + }, + + "exceptions": [ + { + "id": "EX-1", "labelled": true, "requirementId": "R-D1-DELIVERY", + "text": "Delivery windows beginning no earlier than 6:30 AM at all sites; Schedule A specifies several beginning at 5:30 AM.", + "priceDeltaIfDeclined": 18400, + "$comment": "A hard operational conflict with Schedule A, not a preference." + }, + { + "id": "EX-2", "labelled": true, "requirementId": "R-B5-PRICING-TERMS", + "text": "Out-of-region band subject to renegotiation if PPI for fresh vegetables moves more than 12% between award and 1 December.", + "priceDeltaIfDeclined": 9750 + }, + { + "id": "EX-3", "labelled": true, "requirementId": null, + "targetSection": "C.3", + "text": "Pack configuration agreed during onboarding rather than fixed at award, produce categories only.", + "priceDeltaIfDeclined": null, + "isBidWithdrawalCondition": true, + "$comment": "NOT PRICEABLE. If declined the vendor cannot bid produce at all — a conditional withdrawal expressed as an exception. A price-delta field cannot hold it, and a system that models exceptions purely as monetary adjustments loses the most consequential one of the three. It also targets a requirement the RFP truth file does not enumerate (C.3 Specifications), which is itself a finding." + } + ], + + "extractionDifficulty": { + "expectRecall": "medium", + "$comment": "Gates bind reliably via the enclosures paragraph. Narrative requirements bind by section only. The exceptions block is explicitly headed and should extract cleanly — it is the control against which Chesapeake's unlabelled exception is measured." + } +} diff --git a/corpus/friendship-pcs/responses/chesapeake-food-service.truth.json b/corpus/friendship-pcs/responses/chesapeake-food-service.truth.json new file mode 100644 index 0000000..c1f05ca --- /dev/null +++ b/corpus/friendship-pcs/responses/chesapeake-food-service.truth.json @@ -0,0 +1,237 @@ +{ + "$comment": "Ground truth for source/chesapeake-food-service.md. The hard end of the binding gradient.", + "vendor": { + "id": "CHESAPEAKE", + "name": "Chesapeake Food Service, Inc.", + "location": "Baltimore, MD" + }, + "bindingMechanism": { + "level": "free_prose", + "$comment": "#22's least reliable tier. Four sections exist but carry no requirement-level anchors, no RFP references and no matrix. Recovering which sentence answers which requirement is text classification, not structural lookup. This is the document that will expose extraction weakness." + }, + "expectedOutcome": { + "gatesPassed": false, + "failedGates": [ + "R-B6D", + "R-C5-INSPECTIONS" + ], + "atRiskGates": [ + "R-B2-LOCAL", + "R-C1-VALUES", + "R-A2" + ], + "narrative": "Lowest price in the field at $541,280 and DISQUALIFIED regardless, on two missing mandatory health documents. This is the case that proves absence must be representable — a ranking that scores it on price alone puts a disqualified bid at the top." + }, + "answers": [ + { + "requirementId": "R-A1", + "coverage": "answered", + "evidence": "Cover block present" + }, + { + "requirementId": "R-A2", + "coverage": "not_answered", + "evidence": null, + "$comment": "No checklist is reproduced anywhere." + }, + { + "requirementId": "R-B6D", + "coverage": "not_answered", + "evidence": null, + "isGateFailure": true, + "$comment": "THE PRIMARY OMISSION. A current state or local health certificate is never mentioned. Not refused, not deferred — simply absent. The response lists many enclosures, which makes the absence easy for a human to miss and is exactly why coverage must be computed per requirement rather than eyeballed." + }, + { + "requirementId": "R-C5-INSPECTIONS", + "coverage": "not_answered", + "evidence": null, + "isGateFailure": true, + "$comment": "Two most recent health inspections never mentioned. The response DOES discuss pest management at the same facility, which is topically adjacent and will tempt a weak extractor into a false positive." + }, + { + "requirementId": "R-B2-LOCAL", + "coverage": "indeterminate", + "evidence": "'Chesapeake buys from the Maryland and Delmarva growing region in season... We would expect a meaningful proportion of produce delivered between June and October to originate within the region.'", + "$comment": "Addresses the TOPIC warmly and satisfies none of the OBLIGATION — no percentage, no certification, no acceptance of the audit right. This is KM-2's requirement meeting a response that gestures at it. The hardest coverage call in the corpus." + }, + { + "requirementId": "R-C1-VALUES", + "coverage": "indeterminate", + "evidence": "'We are committed to student nutrition and to the communities we serve' — one clause in the Executive Summary, under no heading.", + "$comment": "Exercises KM-3 from the response side. Whether this constitutes a 'response to values statement' is a judgement, not a lookup." + }, + { + "requirementId": "R-B6C", + "coverage": "not_answered", + "evidence": null + }, + { + "requirementId": "R-B7-FORMAT", + "coverage": "answered", + "evidence": "Four sections present", + "$comment": "Section three is titled 'Method of Approach and Implementation', dropping 'Plan'. Near-miss on a required section title — a strict matcher fails, a fuzzy one passes." + }, + { + "requirementId": "R-B8-QUEST", + "coverage": "answered", + "evidence": "'Three references are provided in the questionnaire' — implies completion, never states it" + }, + { + "requirementId": "R-B8-LICENSE", + "coverage": "answered", + "evidence": "'Our business license and certificate of occupancy are enclosed'" + }, + { + "requirementId": "R-B8-OCCUPANCY", + "coverage": "answered", + "evidence": "Same sentence as the business license" + }, + { + "requirementId": "R-B8-FINANCIALS", + "coverage": "answered", + "evidence": "Two years audited, Hanlon & Reeve LLP" + }, + { + "requirementId": "R-D6-INSURANCE", + "coverage": "answered", + "evidence": "$2M per occurrence / $4M aggregate, FPCS as additional insured" + }, + { + "requirementId": "R-ATT-B", + "coverage": "answered", + "evidence": "'debarment, price determination and lobbying certifications are signed and enclosed'" + }, + { + "requirementId": "R-ATT-C", + "coverage": "answered", + "evidence": "Same sentence" + }, + { + "requirementId": "R-ATT-D", + "coverage": "answered", + "evidence": "Same sentence", + "$comment": "THREE separate certification requirements satisfied by ONE sentence naming all three. A per-requirement extractor must split one span three ways." + }, + { + "requirementId": "R-E1-EEO", + "coverage": "answered", + "evidence": "'as are the equal opportunity and clean air and water certifications'" + }, + { + "requirementId": "R-E3-CLEANAIR", + "coverage": "answered", + "evidence": "Same sentence" + }, + { + "requirementId": "R-SCHED-C", + "coverage": "answered", + "evidence": "'submitted separately on the Excel schedule provided'" + }, + { + "requirementId": "R-USDA-FOODS", + "coverage": "answered", + "evidence": "'We participate in the USDA Foods programme in Maryland... our processing relationships are stronger in the protein categories than in fruit and vegetable.'", + "$comment": "ANSWERED but WEAK. Coverage and quality are orthogonal — a correct extraction marks this answered and a correct score marks it low. Conflating them is a defect." + }, + { + "requirementId": "R-CN-LABELS", + "coverage": "answered", + "evidence": "'category managers hold CN labels and formulation statements... typically within two business days of a request'" + }, + { + "requirementId": "R-C6-PEST", + "coverage": "answered", + "evidence": "'licensed applicator, serviced monthly'" + }, + { + "requirementId": "R-D1-DELIVERY", + "coverage": "answered", + "evidence": "Site survey of all eight locations; delivery window confirmed with authorised designee" + }, + { + "requirementId": "R-B5-PRICING-TERMS", + "coverage": "answered", + "evidence": "'Prices are held firm through the base year'" + }, + { + "requirementId": "R-D8-EPA", + "coverage": "answered", + "evidence": "'relief... in the event of a commodity movement exceeding eight percent'" + }, + { + "requirementId": "R-C5-FOODSAFETY", + "coverage": "indeterminate", + "evidence": "Pest management is described; no GFSI, SQF or equivalent certification is claimed.", + "$comment": "Partial. The requirement asks for a food safety programme and the response supplies one component of one." + }, + { + "requirementId": "R-ATT-A", + "coverage": "indeterminate", + "evidence": "'the District's Healthy Schools Act obligations' named in passing, no compliance detail" + }, + { + "requirementId": "R-C7-QA", + "coverage": "not_answered", + "evidence": null + }, + { + "requirementId": "R-C8-FARM2SCHOOL", + "coverage": "not_answered", + "evidence": null + }, + { + "requirementId": "R-C4-MENUS", + "coverage": "not_answered", + "evidence": null + }, + { + "requirementId": "R-D3-RECORDS", + "coverage": "not_answered", + "evidence": null + } + ], + "priceQuote": { + "rateStructure": { + "kind": "flat_unit_plus_frequency_surcharge", + "$comment": "Exercises #29 differently from Potomac. Cost depends on a variable the SFA controls per site and may change mid-term with two weeks' notice — so derived_cost is only defined against a declared frequency assumption, and the response states one.", + "unit": "case", + "flatUnitPrice": null, + "$note": "Unit case price lives only in the external Excel sheet — the KM-1 shape again, this time making the vendor's own stated total unverifiable from the document alone.", + "perStopCharge": [ + { + "deliveriesPerWeek": 2, + "chargePerStop": 34.0 + }, + { + "deliveriesPerWeek": 3, + "chargePerStop": 28.5 + }, + { + "deliveriesPerWeek": 4, + "chargePerStop": 24.0 + } + ], + "frequencyChangeableWithNotice": "2 weeks" + }, + "statedBaseYearTotal": 541280, + "statedAssumptions": { + "deliveriesPerWeek": 3, + "sites": 8, + "calendarDays": 180 + }, + "$verify": "UNVERIFIABLE from this document. The case price is absent, so no computed total can be produced — only the vendor's stated figure exists. Contrast Potomac, where the arithmetic can be checked and fails." + }, + "exceptions": [ + { + "id": "IMPLICIT-1", + "labelled": false, + "requirementId": "R-B5-PRICING-TERMS", + "text": "We would ask only for relief under the economic price adjustment provision in the event of a commodity movement exceeding eight percent", + "$comment": "AN UNLABELLED EXCEPTION. It is phrased as a polite request inside the Pricing section, carries no heading, no price delta and no 'exception' wording. Compare Anacostia, which labels three exceptions explicitly with deltas. #22 called exceptions the thing evaluators care most about and the thing an invented corpus always forgets — this is the version that is present but hidden." + } + ], + "extractionDifficulty": { + "expectRecall": "low", + "$comment": "No anchors of any kind. Expect false positives on R-C5-INSPECTIONS from adjacent pest-management prose, and expect the three-certifications-in-one-sentence span to be split incorrectly or attributed to only one requirement." + } +} diff --git a/corpus/friendship-pcs/responses/potomac-provisions.truth.json b/corpus/friendship-pcs/responses/potomac-provisions.truth.json new file mode 100644 index 0000000..ba9fdce --- /dev/null +++ b/corpus/friendship-pcs/responses/potomac-provisions.truth.json @@ -0,0 +1,90 @@ +{ + "$comment": "Ground truth for source/potomac-provisions.md. Answers key to rfp.truth.json requirement ids.", + "vendor": { "id": "POTOMAC", "name": "Potomac Provisions LLC", "location": "Landover, MD" }, + + "bindingMechanism": { + "level": "tabbed_restatement", + "$comment": "#22's most reliable tier. A compliance matrix restates each RFP reference explicitly, giving a near-guaranteed 1:1 join key. This is the easy end of the gradient and should extract almost perfectly." + }, + + "expectedOutcome": { + "gatesPassed": true, + "narrative": "Complete and responsive. Strongest on USDA Foods and dependability, weakest on geographic preference at 18%. Should win on completeness rather than on any single criterion." + }, + + "answers": [ + { "requirementId": "R-A1", "coverage": "answered", "evidence": "Compliance matrix row A.1 — Complete, signed" }, + { "requirementId": "R-A2", "coverage": "answered", "evidence": "Compliance matrix row A.2" }, + { "requirementId": "R-B2-LOCAL", "coverage": "answered", "evidence": "18% certified; audit right accepted", "value": { "localPercent": 18 } }, + { "requirementId": "R-B6C", "coverage": "answered", "evidence": "Two bid copies provided" }, + { "requirementId": "R-B6D", "coverage": "answered", "evidence": "Prince George's County health certificate, exp. 2022-03-31" }, + { "requirementId": "R-B7-FORMAT", "coverage": "answered", "evidence": "Four titled sections present" }, + { "requirementId": "R-B8-QUEST", "coverage": "answered", "evidence": "Pre-Qualification Questionnaire complete" }, + { "requirementId": "R-B8-LICENSE", "coverage": "answered", "evidence": "MD SDAT D14882910" }, + { "requirementId": "R-B8-OCCUPANCY", "coverage": "answered", "evidence": "Attached" }, + { "requirementId": "R-B8-FINANCIALS", "coverage": "answered", "evidence": "FY2019 and FY2020 audited" }, + { "requirementId": "R-C1-VALUES", "coverage": "answered", "evidence": "Section Four, 'Response to values statement' — explicit heading" }, + { "requirementId": "R-C5-INSPECTIONS", "coverage": "answered", "evidence": "2020-11-04 and 2021-02-17, both passed" }, + { "requirementId": "R-D6-INSURANCE", "coverage": "answered", "evidence": "$2M per occurrence / $4M aggregate" }, + { "requirementId": "R-SCHED-C", "coverage": "answered", "evidence": "Submitted in Excel as instructed; structure summarised in Section Two", + "$comment": "The RESPONSE references pricing correctly, but the actual price sheet is outside this document — the same KM-1 shape on the response side." }, + { "requirementId": "R-ATT-A", "coverage": "answered", "evidence": "HSA acknowledged, Section Three" }, + { "requirementId": "R-ATT-B", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-ATT-C", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-ATT-D", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-E1-EEO", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-E3-CLEANAIR", "coverage": "answered", "evidence": "Signed" }, + { "requirementId": "R-C5-FOODSAFETY", "coverage": "answered", "evidence": "GFSI-certified under SQF Level 2" }, + { "requirementId": "R-C6-PEST", "coverage": "answered", "evidence": "Licensed applicator, monthly service, quarterly third-party audit" }, + { "requirementId": "R-C7-QA", "coverage": "answered", "evidence": "USDA No. 1 or better; driver authorised to credit at receipt" }, + { "requirementId": "R-C8-FARM2SCHOOL", "coverage": "indeterminate", "evidence": "Local sourcing addressed at 18%, but Farm to School participation as such is never mentioned", + "$comment": "The topic is adjacent and the requirement is not squarely answered. This is the coverage state that most needs a human." }, + { "requirementId": "R-USDA-FOODS", "coverage": "answered", "evidence": "Approved processor-distributor MD and DC; 340,000 lbs processed SY2020-21" }, + { "requirementId": "R-CN-LABELS", "coverage": "answered", "evidence": "Portal per-item document library; bulk PDF per menu cycle" }, + { "requirementId": "R-D1-DELIVERY", "coverage": "answered", "evidence": "Freight to all eight sites within Schedule A windows" }, + { "requirementId": "R-D8-EPA", "coverage": "answered", "evidence": "PPI-based, at contract anniversary only" }, + { "requirementId": "R-B5-PRICING-TERMS", "coverage": "answered", "evidence": "Prices firm for base year" }, + { "requirementId": "R-D3-RECORDS", "coverage": "not_answered", "evidence": null, + "$comment": "Narrative requirement, not a gate — no disqualification, but it is a real gap in an otherwise complete response." }, + { "requirementId": "R-C4-MENUS", "coverage": "not_answered", "evidence": null } + ], + + "priceQuote": { + "rateStructure": { + "kind": "volume_banded", + "$comment": "Exercises #29. Retroactive year-end rebate — the applicable band is not knowable at order time, only at year end, which means derived_cost must be computed against the DECLARED scenario volume rather than observed.", + "unit": "case", + "listPrice": 28.40, + "bands": [ + { "minCases": 0, "maxCases": 11999, "discountPercent": 0.0, "effectiveUnitPrice": 28.40 }, + { "minCases": 12000, "maxCases": 23999, "discountPercent": 3.0, "effectiveUnitPrice": 27.55 }, + { "minCases": 24000, "maxCases": 35999, "discountPercent": 5.5, "effectiveUnitPrice": 26.84 }, + { "minCases": 36000, "maxCases": null, "discountPercent": 7.5, "effectiveUnitPrice": 26.27 } + ], + "freightIncluded": true, + "fuelSurcharge": false, + "minimumOrderCharge": false + }, + "statedBaseYearTotal": 589970, + "statedAtScenarioVolume": 21400, + "$verify": "21400 cases falls in band 2 (12000-23999) at 27.55 -> 21400 * 27.55 = 589,570. The document states 589,970. DELIBERATE 400 DISCREPANCY — see arithmeticTraps." + }, + + "exceptions": [], + + "arithmeticTraps": [ + { + "id": "AT-1", + "where": "Section Two, base year total", + "$comment": "The stated total ($589,970) does not equal band price x scenario volume ($589,570). A $400 gap. Planted deliberately: #15 decision 2 puts all price arithmetic in code precisely so a stated total is never trusted over a computed one. An extractor that lifts the vendor's stated figure reports the wrong number and nothing catches it.", + "statedTotal": 589970, + "computedTotal": 589570, + "delta": 400 + } + ], + + "extractionDifficulty": { + "expectRecall": "very high", + "$comment": "The compliance matrix gives explicit RFP references. Failure here indicates a table-parsing defect rather than a binding problem." + } +} diff --git a/corpus/friendship-pcs/responses/source/anacostia-farm-collective.md b/corpus/friendship-pcs/responses/source/anacostia-farm-collective.md new file mode 100644 index 0000000..5881d8a --- /dev/null +++ b/corpus/friendship-pcs/responses/source/anacostia-farm-collective.md @@ -0,0 +1,184 @@ +# ANACOSTIA FARM COLLECTIVE +## Response to RFP — Supply and Delivery of Food Items +### Friendship Public Charter School, SY 2021–2022 + +2214 Martin Luther King Jr Ave SE, Washington, DC 20020 +Yolanda Reyes, Managing Director — (202) 555-0173 + +--- + +## Section One — Executive Summary + +Anacostia Farm Collective is a District-based food hub aggregating from +thirty-four farms within the Chesapeake foodshed. We were formed in 2014 +specifically to make regionally grown food available to institutions that +want it and cannot reach it through conventional broadline distribution. +We currently supply eleven institutional accounts including four DC public +charter schools. + +We are the smallest bidder Friendship is likely to consider, and we want to +be direct about what that means. We will not be the cheapest bid. What we +offer is the highest local content available in this market and a supply +relationship that is legible all the way back to the farm. + +**Enclosures.** Business license (DC BBL 410203000891); Certificate of +Occupancy; audited financial statements FY2019 and FY2020, prepared by +Mercer Group CPAs; current DC Department of Health food establishment +certificate, expiring 2022-06-30; our two most recent health inspections, +dated 2020-09-22 and 2021-01-14, both passed; certificate of insurance; +Attachments B, C and D signed; equal opportunity and clean air and water +certifications signed; certification of local percentage under B.2. + +**Exceptions.** We take three exceptions to the solicitation as issued. +They are set out in full at the end of this response with the price effect +of each. We would rather state them plainly now than discover them at +contract execution. + +--- + +## Section Two — Pricing + +Submitted on Schedule C in the Excel workbook provided. + +**Rate structure: seasonal piecewise unit pricing.** Because our supply is +regional and genuinely seasonal, a single annual unit price would require +us to carry the winter cost of out-of-region sourcing across the whole +year. Instead we price produce in three seasonal bands. Non-produce +categories — grocery, dairy, protein — are priced flat. + +| Band | Months | Produce index | Basis | +|---|---|---|---| +| Peak regional | June – October | 0.88 × base | in-region supply at volume | +| Shoulder | April – May, November | 1.00 × base | mixed regional and contracted | +| Out-of-region | December – March | 1.19 × base | sourced outside foodshed | + +Base delivered produce case price, mixed: $31.20. Grocery, dairy and +protein are quoted flat on Schedule C. + +Applying the band weighting across the 180-day school calendar — which +falls disproportionately in the shoulder and out-of-region bands — our +**base year total bid cost is $624,315.** + +We recognise this is materially above the likely low bid, and roughly six +percent above what we would guess the middle of this field looks like. We +believe the local content justifies it against the Geographic Preference +criterion and against the Healthy Schools Act local provisions, and we +would rather present an honest seasonal structure than a flat price that +conceals the same economics. + +Prices firm for the base year within the stated bands. + +--- + +## Section Three — Method of Approach and Implementation Plan + +### Food and packaging + +Produce is packed to USDA No. 1. Because we aggregate from multiple farms, +pack configuration varies more than a broadline distributor's would, and we +would agree standard configurations with the SFA during onboarding rather +than assume them. Every case is labelled with the originating farm. + +Quality assurance is farm-level: each member farm carries GAP certification +and we audit annually. Any item rejected at receipt is credited on the +delivery ticket. + +CN labels and formulation statements are held for all processed and grocery +items and are provided through our shared drive per account. Whole produce +does not carry CN labels; for those items we supply the crediting +documentation the meal pattern requires. + +### Utilization of USDA Foods + +This is our weakest area and we will not overstate it. We are not currently +an approved USDA Foods processor. We would work alongside a processor +designated by the SFA, or support the SFA in managing entitlement directly, +but we cannot manage drawdown and diversion in-house today. + +### Geographic preference + +**Sixty-one percent (61%)** of the product value we propose to supply will +be locally grown or raised within 250 miles of Washington, DC, measured +across the full contract year. In the peak regional band that figure +exceeds eighty percent. Certification of the percentage is enclosed under +B.2 and we accept the SFA's audit right. + +All thirty-four member farms are within the Chesapeake foodshed; nineteen +are in Maryland, nine in Virginia, four in Delaware and two within the +District itself. + +### Healthy Schools Act + +We supply four DC charter SFAs under HSA requirements today and our +certification of local percentage is designed to support the SFA's local +sourcing subsidy claim directly. + +--- + +## Section Four — Bidder's Experience, Expertise and Reliability + +**Knowledge of USDA and District requirements.** Our operations lead spent +six years in DC OSSE child nutrition administration. We are fluent in the +HSA and in the meal pattern; we are less fluent in USDA Foods entitlement +mechanics, as stated above. + +**Dependability.** On-time delivery for SY2020–21 was 96.1% against +contracted windows, and order fill accuracy 97.4% by line. Both are below +what a broadline distributor will quote. The gap is almost entirely +weather-driven crop shortfall in the peak band, where we substitute within +category rather than fail the line. + +**Warehouse and facilities.** Our aggregation facility on Martin Luther +King Jr Avenue is DC Department of Health certified. Two most recent +inspections enclosed, both passed. Pest management is contracted monthly +with a licensed applicator and audited semi-annually. + +**Customer service.** Named account lead, orders by phone, email or shared +order sheet until 3:00 PM for next-day. The managing director is reachable +directly and is on the escalation path for every account. + +**Proven organizational capacity.** Seven years in operation, eleven +institutional accounts, thirty-four member farms. Our largest current +account is three sites. **Friendship's eight sites would be the largest +network we have served**, and we would staff a dedicated route and a +second driver for it, funded within the pricing above. + +**Response to values statement (C.1).** Friendship's values statement +speaks to educational outcomes and to the community the school sits in. We +are based in Ward 8, we employ from Ward 8, and two of our member farms are +inside the District. When Friendship spends with us, a measurable share of +that spend stays within the same community the students come from. That is +the whole reason the Collective exists, and it is why we are bidding +despite being the highest price on the table. + +**References.** Three DC institutional accounts, provided in the +questionnaire. + +--- + +## Exceptions and Qualifications + +We take the following three exceptions to the solicitation as issued. + +**Exception 1 — D.1 Delivery Requirements, delivery windows.** +Schedule A specifies delivery windows at several sites beginning at 5:30 +AM. Our aggregation model consolidates farm deliveries overnight and we +cannot reliably load before 5:00 AM. We request windows beginning no +earlier than 6:30 AM at all sites. +*Price effect if not granted: +$18,400 (second overnight shift).* + +**Exception 2 — B.5, price firmness across the full term.** +We request that the out-of-region band (December–March) be subject to +renegotiation if the published PPI for fresh vegetables moves more than +twelve percent between award and 1 December. Our regional supply cannot +absorb that movement the way a national distributor's can. +*Price effect if not granted: +$9,750 (risk premium carried in base).* + +**Exception 3 — C.3 Specifications, pack configuration.** +We request that pack configuration be agreed jointly during onboarding +rather than fixed at award, for produce categories only. Aggregated supply +cannot guarantee a single fixed pack across thirty-four farms. +*Price effect if not granted: we would be unable to bid produce categories.* + +Total price effect of exceptions 1 and 2 if declined: **+$28,150**, giving +an adjusted base year total bid cost of **$652,465**. diff --git a/corpus/friendship-pcs/responses/source/chesapeake-food-service.md b/corpus/friendship-pcs/responses/source/chesapeake-food-service.md new file mode 100644 index 0000000..89333b3 --- /dev/null +++ b/corpus/friendship-pcs/responses/source/chesapeake-food-service.md @@ -0,0 +1,118 @@ +# Chesapeake Food Service, Inc. +## Proposal to Friendship Public Charter School — Food Items, 2021–2022 + +1120 Ridgely Street, Baltimore, MD 21230 +Marcus Bell, Vice President — (410) 555-0188 + +--- + +### Executive Summary + +Chesapeake Food Service is a family-owned broadline distributor operating +out of Baltimore since 1978. We serve roughly two hundred accounts across +Maryland and the District, of which about thirty are schools, and we are +pleased to offer Friendship Public Charter School the most competitive +delivered pricing in this market. + +Our proposition is straightforward. We buy well, we run lean, and we pass +that through. Where other distributors in this bid will layer on portal +technology and account management structures that ultimately appear in the +case price, we have chosen to compete on the number that matters. Over an +eight-site network across a full school year, the difference is +substantial, and it is money that returns directly to the meal programme. + +Our business license and certificate of occupancy are enclosed, along with +two years of audited financial statements prepared by Hanlon & Reeve LLP. +The debarment, price determination and lobbying certifications are signed +and enclosed, as are the equal opportunity and clean air and water +certifications. Insurance is carried at two million per occurrence and four +million aggregate and a certificate naming Friendship Public Charter School +as additional insured is enclosed. + +We are committed to student nutrition and to the communities we serve, and +we believe strongly that a school food programme succeeds or fails on +consistency of supply rather than on marketing. That is the value we bring. + +### Pricing + +Our pricing is submitted separately on the Excel schedule provided. + +We price at a flat delivered unit cost per case with no volume tiers and no +rebate mechanism, because we have found that banded pricing chiefly serves +to make comparison difficult and to defer benefit to year end. Our number +is our number from day one. + +Delivered case pricing is applied as quoted plus a per-stop delivery charge +which varies with the frequency requested. At two deliveries per week per +site, the charge is $34.00 per stop. At three per week it falls to $28.50 +per stop, and at four or more it falls to $24.00 per stop. Sites may elect +different frequencies and may change frequency with two weeks' notice. + +At the estimated volume stated in the solicitation and assuming three +deliveries weekly to each of the eight sites across a one hundred and +eighty day calendar, our base year total bid cost is five hundred and +forty-one thousand, two hundred and eighty dollars ($541,280). + +Prices are held firm through the base year. We would ask only for relief +under the economic price adjustment provision in the event of a commodity +movement exceeding eight percent, on the same published index basis +contemplated in the solicitation. + +### Method of Approach and Implementation + +We would begin with a site survey of all eight locations in the two weeks +following award, walking each receiving area with the site's authorised +designee to confirm the delivery window, the receiving point and the +storage capacity. Our experience is that most transition problems are +receiving problems rather than supply problems, and they are best solved +before the first delivery rather than after. + +Product is packed to USDA standards throughout. We have not historically +found that schools require pack size changes from our standard +configurations, but we would accommodate reasonable requests where our +suppliers can support them. + +On nutritional documentation, our category managers hold CN labels and +formulation statements for the school product range and provide these on +request. We can supply these for any item in the proposed catalogue, +typically within two business days of a request. + +We participate in the USDA Foods programme in Maryland and would work with +the SFA to make appropriate use of entitlement, though we would note that +our processing relationships are stronger in the protein categories than +in fruit and vegetable. We think it right to be plain about that rather +than to overstate it. + +As to local sourcing, Chesapeake buys from the Maryland and Delmarva +growing region in season and has done so throughout its history. We are +proud of those relationships. We would expect a meaningful proportion of +produce delivered between June and October to originate within the region. + +We maintain a pest management programme at our Baltimore facility under +contract with a licensed applicator, serviced monthly. + +### Bidder's Experience, Expertise and Reliability + +Forty-three years in continuous operation under the same family ownership +is, we would suggest, its own evidence of dependability. We have never +failed to complete work awarded to us and we have never been terminated for +cause. + +We deliver on time. Our internal measurement for the last school year was +in the high nineties by percentage, and we are comfortable being held to +that standard contractually. + +Our staff know the USDA meal pattern requirements and the District's +Healthy Schools Act obligations, and several of our account team have +worked with DC schools for more than a decade. + +We would provide a single point of contact for the network with escalation +directly to the undersigned. We do not operate an ordering portal; orders +are placed by telephone or email to a named person, which our customers +tell us they prefer. + +Three references are provided in the questionnaire, all Maryland school +accounts. + +We appreciate the opportunity and would welcome the chance to discuss any +element of this proposal with the committee. diff --git a/corpus/friendship-pcs/responses/source/potomac-provisions.md b/corpus/friendship-pcs/responses/source/potomac-provisions.md new file mode 100644 index 0000000..04090fb --- /dev/null +++ b/corpus/friendship-pcs/responses/source/potomac-provisions.md @@ -0,0 +1,170 @@ +# POTOMAC PROVISIONS LLC +### Response to Friendship Public Charter School +### RFP for the Supply and Delivery of Food Items, SY 2021–2022 + +8600 Ardwick-Ardmore Road, Landover, MD 20785 · (301) 555-0142 +Contact: Denise Whitaker, Director of Education Accounts + +--- + +## SECTION ONE — EXECUTIVE SUMMARY + +Potomac Provisions has distributed to School Food Authorities in the +Washington metropolitan area since 1994. We currently serve 41 SFAs across +DC, Maryland and Northern Virginia, including 11 public charter networks, +and deliver approximately 2.3 million cases annually from a single +GFSI-certified warehouse in Landover. + +### Compliance Matrix + +Per Section A.2, every required item is enumerated below with the page of +this response on which it appears. + +| RFP Ref | Requirement | Status | Page | +|---|---|---|---| +| A.1 | Cover Page | Complete, signed | 1 | +| A.2 | Bidder Checklist | Complete | 2 | +| B.2 | Certification of percentage of local foods | Complete — 18% certified | 3 | +| B.6(c) | Two bid copies | Provided | — | +| B.6(d) | Current state or local health certificate | Attached, Prince George's County, exp. 2022-03-31 | A-1 | +| B.7 | Four-section format | This document | — | +| B.8 | Distributor Pre-Qualification Questionnaire | Complete | 4 | +| B.8 | Business license | Attached, MD SDAT D14882910 | A-2 | +| B.8 | Certificate of Occupancy | Attached | A-3 | +| B.8 | Two years audited financial statements | Attached, FY2019 & FY2020 | A-4 | +| C.1 | Response to values statement | Section Four | 7 | +| C.5 | Two most recent health inspections | Attached, 2020-11-04 and 2021-02-17, both passed | A-5 | +| D.6 | Insurance amounts | Certificate attached, $2M/$4M | A-6 | +| Schedule C | Pricing Sheet | Submitted in Excel as instructed | — | +| Att. A | Healthy Schools Act requirements | Acknowledged, Section Three | 5 | +| Att. B | Debarment & Suspension | Signed | A-7 | +| Att. C | Independent Price Determination | Signed | A-8 | +| Att. D | Certification Regarding Lobbying | Signed | A-9 | +| E.1 | Equal Opportunity | Signed | A-10 | +| E.3 | Clean Air and Water Certification | Signed | A-11 | + +No exceptions are taken to any term or condition of this solicitation. + +--- + +## SECTION TWO — PRICING + +Pricing is submitted on Schedule C in the Excel workbook provided, as +instructed. The structure is summarised here for the evaluation committee. + +**Rate structure: volume-banded unit pricing.** Unit price per case falls as +aggregate annual volume across all eight sites rises. Bands are measured on +total cases invoiced in the contract year and applied retroactively at +year end as a rebate. + +| Annual case volume | Discount from list | Illustrative delivered case price, mixed produce | +|---|---|---| +| 0 – 11,999 cases | list | $28.40 | +| 12,000 – 23,999 cases | 3.0% | $27.55 | +| 24,000 – 35,999 cases | 5.5% | $26.84 | +| 36,000+ cases | 7.5% | $26.27 | + +Delivered price includes freight to all eight sites within the delivery +windows in Schedule A. There is no separate fuel surcharge and no minimum +order charge. + +**Base year total bid cost, at FPCS's stated estimated volume of 21,400 +cases: $589,970.** + +Prices are firm for the base year. Economic price adjustment is requested +only as permitted under D.8, on the published PPI for the relevant +commodity group, and only at contract anniversary. + +--- + +## SECTION THREE — METHOD OF APPROACH AND IMPLEMENTATION PLAN + +### Food and packaging + +All produce is packed to USDA No. 1 or better. Case configurations follow +the SFA's existing pack sizes so that no menu re-costing is required at +transition. Product acceptance issues are credited on the delivery ticket +at the point of receipt; the driver is authorised to credit without a +return authorisation. + +CN labels and product formulation statements are supplied through our +online portal, which exposes a per-item document library. Every catalogue +item carries its CN label, nutrient analysis and allergen statement, and +these are downloadable in bulk as a single PDF per menu cycle. This is how +we make nutritional information available to the SFA as required in B.7. + +### Utilization of USDA Foods + +We are an approved processor-distributor for USDA Foods in both Maryland +and the District. We manage entitlement drawdown, diversion and pass-through +value directly, and provide a monthly entitlement position statement. +For SY2020–21 we processed 340,000 lbs of USDA-donated commodity on behalf +of DC-area SFAs. + +### Geographic preference + +Eighteen percent (18%) of the product value we propose to supply to FPCS +will be locally grown or raised, defined as originating within 250 miles +of Washington, DC. Certification of this percentage is provided under B.2. +We accept the SFA's right to audit the actual percentage once awarded. + +We are candid that 18% is a function of our scale and our year-round +service commitment: we will not substitute a local item where continuity +of supply cannot be guaranteed across the full school year. + +### Healthy Schools Act + +We have supplied DC SFAs under HSA requirements since 2010 and are familiar +with the local meal reimbursement conditions in Attachment A, including the +whole-grain-rich and local-sourcing provisions. + +--- + +## SECTION FOUR — BIDDER'S EXPERIENCE, EXPERTISE AND RELIABILITY + +### Knowledge of USDA and District requirements + +Our education accounts team includes two staff who previously worked in +SFA operations. We complete DC OSSE reporting alongside our SFA customers +each year. + +### Dependability + +Our on-time delivery rate across all SFA accounts for SY2020–21 was 98.6%, +measured against contracted delivery windows. Order fill accuracy was +99.2% by line. Both figures are drawn from our WMS and can be audited. + +### Warehouse and facilities + +Our Landover facility is GFSI-certified under SQF Level 2. The two most +recent health inspections, both passed with no critical violations, are +attached under C.5. Our stored products pest management programme is +operated under contract by a licensed applicator with monthly service and +quarterly third-party audit, per C.6. + +### Customer service + +Each SFA has a named account manager and a named backup. Orders may be +placed through the portal or by phone until 2:00 PM for next-day delivery. +Credits are issued within one business day. + +### Proven organizational capacity + +Twenty-seven years in operation. 2.3 million cases annually. We have run +eight-site or larger charter networks continuously since 2011, and our +audited financials for FY2019 and FY2020 are attached. + +### Response to values statement (C.1) + +FPCS's stated commitment to student nutrition as a driver of educational +outcomes matches how we run our education accounts. We staff them +separately from our commercial business precisely because the buying +calendar, the compliance burden and the tolerance for substitution are +different. We are interested in this business because eight sites under one +authority is the scale at which menu consistency actually becomes +achievable. + +### References + +Three SFA references are provided in the Pre-Qualification Questionnaire, +all DC or Prince George's County charter networks of comparable size. diff --git a/corpus/friendship-pcs/rfp.truth.json b/corpus/friendship-pcs/rfp.truth.json index 7f8d3cf..39b3e27 100644 --- a/corpus/friendship-pcs/rfp.truth.json +++ b/corpus/friendship-pcs/rfp.truth.json @@ -6,7 +6,6 @@ "", "Requirement ids are stable and are what response truth files answer against." ], - "solicitation": { "title": "Request for Proposal for the Supply and Delivery of Food Items", "issuer": "Friendship Public Charter School", @@ -14,10 +13,16 @@ "issueDate": "2021-03-15", "closingDate": "2021-04-16", "pages": 52, - "programContext": ["NSLP", "SBP", "ASSP", "FFVP", "CACFP", "SFSP"], + "programContext": [ + "NSLP", + "SBP", + "ASSP", + "FFVP", + "CACFP", + "SFSP" + ], "siteCount": 8 }, - "bindingMechanism": { "level": "named_section", "$comment": [ @@ -27,42 +32,151 @@ "This is the single most important fact about scoring this document." ], "requiredSections": [ - { "ordinal": 1, "title": "Executive Summary", "mapsToCriterion": null }, - { "ordinal": 2, "title": "Pricing", "mapsToCriterion": "PRICING" }, - { "ordinal": 3, "title": "Method of Approach and Implementation Plan", "mapsToCriterion": "METHOD" }, - { "ordinal": 4, "title": "Bidder's Experience, Expertise and Reliability", "mapsToCriterion": "EXPERIENCE" } + { + "ordinal": 1, + "title": "Executive Summary", + "mapsToCriterion": null + }, + { + "ordinal": 2, + "title": "Pricing", + "mapsToCriterion": "PRICING" + }, + { + "ordinal": 3, + "title": "Method of Approach and Implementation Plan", + "mapsToCriterion": "METHOD" + }, + { + "ordinal": 4, + "title": "Bidder's Experience, Expertise and Reliability", + "mapsToCriterion": "EXPERIENCE" + } ] }, - "ratingScale": { "$comment": "Per-document, never normalised (#10, #22). The prose anchor is load-bearing, not the integer.", "min": 0, "max": 5, "anchors": [ - { "points": 0, "label": "Completely Deficient", "anchor": "Elements is missing or is completely nonresponsive to the request" }, - { "points": 1, "label": "Unsatisfactory", "anchor": "Fails to meet minimum requirements; major deficiencies which are not correctable" }, - { "points": 2, "label": "Below Average", "anchor": "Marginally meets minimum requirements; significant deficiencies which may be correctable" }, - { "points": 3, "label": "Average", "anchor": "Meets requirements; only minor deficiencies which are correctable" }, - { "points": 4, "label": "Very Good", "anchor": "Meets requirements; no deficiencies" }, - { "points": 5, "label": "Exceptional", "anchor": "Exceeds most, if not all requirements; no deficiencies" } + { + "points": 0, + "label": "Completely Deficient", + "anchor": "Elements is missing or is completely nonresponsive to the request" + }, + { + "points": 1, + "label": "Unsatisfactory", + "anchor": "Fails to meet minimum requirements; major deficiencies which are not correctable" + }, + { + "points": 2, + "label": "Below Average", + "anchor": "Marginally meets minimum requirements; significant deficiencies which may be correctable" + }, + { + "points": 3, + "label": "Average", + "anchor": "Meets requirements; only minor deficiencies which are correctable" + }, + { + "points": 4, + "label": "Very Good", + "anchor": "Meets requirements; no deficiencies" + }, + { + "points": 5, + "label": "Exceptional", + "anchor": "Exceeds most, if not all requirements; no deficiencies" + } ] }, - "criteria": [ - { "id": "PRICING", "parentId": null, "label": "Pricing", "maxPoints": 40, "page": 37 }, - { "id": "METHOD", "parentId": null, "label": "Method of Approach and Implementation", "maxPoints": 30, "page": 37 }, - { "id": "FOOD_PACKAGING", "parentId": "METHOD", "label": "Food and Packaging (Appearance, Quality, Taste and Acceptance)", "maxPoints": 15, "page": 37 }, - { "id": "USDA_FOODS", "parentId": "METHOD", "label": "Utilization of USDA Foods", "maxPoints": 10, "page": 37 }, - { "id": "GEO_PREF", "parentId": "METHOD", "label": "Geographic preference — locally grown or raised foods", "maxPoints": 5, "page": 37 }, - { "id": "EXPERIENCE", "parentId": null, "label": "Bidder's Experience, Expertise, & Reliability", "maxPoints": 20, "page": 37 }, - { "id": "EXP_KNOWLEDGE", "parentId": "EXPERIENCE", "label": "Knowledge of USDA, District requirements", "maxPoints": 5, "page": 37 }, - { "id": "EXP_DEPENDABILITY", "parentId": "EXPERIENCE", "label": "Dependability (on-time deliveries, accuracy filling orders)", "maxPoints": 5, "page": 37 }, - { "id": "EXP_WAREHOUSE", "parentId": "EXPERIENCE", "label": "Inspection of warehouse and other facilities", "maxPoints": 5, "page": 37 }, - { "id": "EXP_SERVICE", "parentId": "EXPERIENCE", "label": "Customer Service", "maxPoints": 5, "page": 37 }, - { "id": "ORG_CAPACITY", "parentId": null, "label": "Proven Organizational Capacity", "maxPoints": 5, "page": 38 }, - { "id": "COMMUNITY", "parentId": null, "label": "Community Connection, Values Alignment", "maxPoints": 5, "page": 38 } + { + "id": "PRICING", + "parentId": null, + "label": "Pricing", + "maxPoints": 40, + "page": 37 + }, + { + "id": "METHOD", + "parentId": null, + "label": "Method of Approach and Implementation", + "maxPoints": 30, + "page": 37 + }, + { + "id": "FOOD_PACKAGING", + "parentId": "METHOD", + "label": "Food and Packaging (Appearance, Quality, Taste and Acceptance)", + "maxPoints": 15, + "page": 37 + }, + { + "id": "USDA_FOODS", + "parentId": "METHOD", + "label": "Utilization of USDA Foods", + "maxPoints": 10, + "page": 37 + }, + { + "id": "GEO_PREF", + "parentId": "METHOD", + "label": "Geographic preference — locally grown or raised foods", + "maxPoints": 5, + "page": 37 + }, + { + "id": "EXPERIENCE", + "parentId": null, + "label": "Bidder's Experience, Expertise, & Reliability", + "maxPoints": 20, + "page": 37 + }, + { + "id": "EXP_KNOWLEDGE", + "parentId": "EXPERIENCE", + "label": "Knowledge of USDA, District requirements", + "maxPoints": 5, + "page": 37 + }, + { + "id": "EXP_DEPENDABILITY", + "parentId": "EXPERIENCE", + "label": "Dependability (on-time deliveries, accuracy filling orders)", + "maxPoints": 5, + "page": 37 + }, + { + "id": "EXP_WAREHOUSE", + "parentId": "EXPERIENCE", + "label": "Inspection of warehouse and other facilities", + "maxPoints": 5, + "page": 37 + }, + { + "id": "EXP_SERVICE", + "parentId": "EXPERIENCE", + "label": "Customer Service", + "maxPoints": 5, + "page": 37 + }, + { + "id": "ORG_CAPACITY", + "parentId": null, + "label": "Proven Organizational Capacity", + "maxPoints": 5, + "page": 38 + }, + { + "id": "COMMUNITY", + "parentId": null, + "label": "Community Connection, Values Alignment", + "maxPoints": 5, + "page": 38 + } ], - "criteriaNotes": { "totalPoints": 100, "nestingIsInconsistent": { @@ -73,8 +187,14 @@ "they are requirements mapped to the criterion, not sub-criteria. Identical visual", "nesting, two different meanings, in one document." ], - "weightedParents": ["METHOD", "EXPERIENCE"], - "guidanceOnlyParents": ["ORG_CAPACITY", "COMMUNITY"] + "weightedParents": [ + "METHOD", + "EXPERIENCE" + ], + "guidanceOnlyParents": [ + "ORG_CAPACITY", + "COMMUNITY" + ] }, "scoringLevelIsAmbiguous": { "$comment": [ @@ -86,48 +206,307 @@ ] } }, - "requirements": [ - { "id": "R-A1", "kind": "mandatory_gate", "section": "A.1", "page": 4, "text": "Cover Page completed and signed.", "criterionIds": [] }, - { "id": "R-A2", "kind": "mandatory_gate", "section": "A.2", "page": 5, "text": "Bidder Checklist completed.", "criterionIds": [] }, - - { "id": "R-B2-LOCAL", "kind": "mandatory_gate", "section": "B.2", "page": 5, "text": "Certification of percentage of local foods to be provided.", "criterionIds": ["GEO_PREF"], "knownMissId": "KM-2" }, - { "id": "R-B6C", "kind": "mandatory_gate", "section": "B.6(c)", "page": 5, "text": "Two bid copies submitted.", "criterionIds": [] }, - { "id": "R-B6D", "kind": "mandatory_gate", "section": "B.6(d)", "page": 5, "text": "A copy of a current state or local health certificate assuring that all regulations for the preparation of food and the facilities are being met.", "criterionIds": ["EXP_WAREHOUSE"] }, - { "id": "R-B8-QUEST", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Distributor Pre-Qualification Questionnaire completed.", "criterionIds": ["EXP_KNOWLEDGE"] }, - { "id": "R-B8-LICENSE", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Business license.", "criterionIds": [] }, - { "id": "R-B8-OCCUPANCY", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Certificate of Occupancy.", "criterionIds": [] }, - { "id": "R-B8-FINANCIALS", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Two years of audited financial statements.", "criterionIds": ["ORG_CAPACITY"] }, - - { "id": "R-C1-VALUES", "kind": "mandatory_gate", "section": "C.1", "page": 5, "text": "Response to values statement.", "criterionIds": ["COMMUNITY"], "knownMissId": "KM-3" }, - { "id": "R-C5-INSPECTIONS", "kind": "mandatory_gate", "section": "C.5", "page": 5, "text": "Two most recent health inspections.", "criterionIds": ["EXP_WAREHOUSE"] }, - { "id": "R-D6-INSURANCE", "kind": "mandatory_gate", "section": "D.6", "page": 5, "text": "Insurance amounts.", "criterionIds": [] }, - - { "id": "R-SCHED-C", "kind": "pricing", "section": "Schedule C", "page": 41, "text": "Pricing Sheet — separate attachment to be completed in Excel.", "criterionIds": ["PRICING"], "knownMissId": "KM-1" }, - - { "id": "R-ATT-B", "kind": "certification_form", "section": "Attachment B", "page": 44, "text": "Certificate Regarding Debarment, Suspension, Ineligibility and Voluntary Exclusion.", "criterionIds": [], "externalFormSchema": true }, - { "id": "R-ATT-C", "kind": "certification_form", "section": "Attachment C", "page": 46, "text": "Certificate of Independent Price Determination.", "criterionIds": [], "externalFormSchema": true }, - { "id": "R-ATT-D", "kind": "certification_form", "section": "Attachment D", "page": 47, "text": "Permanent Certification Regarding Lobbying.", "criterionIds": [], "externalFormSchema": true }, - { "id": "R-ATT-A", "kind": "compliance_schedule", "section": "Attachment A", "page": 42, "text": "Healthy Schools Act requirements compliance.", "criterionIds": ["EXP_KNOWLEDGE"] }, - - { "id": "R-B7-FORMAT", "kind": "mandatory_gate", "section": "B.7", "page": 15, "text": "The submitted proposal must have four sections, titled: Executive Summary; Pricing; Method of Approach and Implementation Plan; Bidder's Experience, Expertise and Reliability.", "criterionIds": [] }, - - { "id": "R-C5-FOODSAFETY", "kind": "narrative", "section": "C.5", "page": 23, "text": "Warehouse and distribution food safety programme.", "criterionIds": ["EXP_WAREHOUSE", "FOOD_PACKAGING"] }, - { "id": "R-C6-PEST", "kind": "narrative", "section": "C.6", "page": 24, "text": "Stored products pest management programme.", "criterionIds": ["EXP_WAREHOUSE"] }, - { "id": "R-C7-QA", "kind": "narrative", "section": "C.7", "page": 24, "text": "Distributor's responsibility to provide quality assurance.", "criterionIds": ["FOOD_PACKAGING"] }, - { "id": "R-C8-FARM2SCHOOL", "kind": "narrative", "section": "C.8", "page": 25, "text": "Farm to School participation.", "criterionIds": ["GEO_PREF", "COMMUNITY"] }, - { "id": "R-C4-MENUS", "kind": "narrative", "section": "C.4", "page": 23, "text": "Use of advisory group and menus.", "criterionIds": ["EXP_SERVICE"] }, - { "id": "R-D1-DELIVERY", "kind": "narrative", "section": "D.1", "page": 26, "text": "Delivery requirements, including delivery windows per school site.", "criterionIds": ["EXP_DEPENDABILITY"] }, - { "id": "R-D3-RECORDS", "kind": "narrative", "section": "D.3", "page": 27, "text": "Record-keeping obligations.", "criterionIds": ["EXP_KNOWLEDGE"] }, - { "id": "R-USDA-FOODS", "kind": "narrative", "section": "B.2/C.3", "page": 7, "text": "Utilization of USDA Foods in the product offering, including processing and diversion.", "criterionIds": ["USDA_FOODS"] }, - { "id": "R-CN-LABELS", "kind": "narrative", "section": "B.7", "page": 15, "text": "How the Distributor will make nutritional information such as CN labels available to the SFA.", "criterionIds": ["FOOD_PACKAGING", "EXP_KNOWLEDGE"] }, - - { "id": "R-D8-EPA", "kind": "narrative", "section": "D.8", "page": 30, "text": "Economic price adjustment and price renegotiation terms.", "criterionIds": ["PRICING"] }, - { "id": "R-B5-PRICING-TERMS", "kind": "narrative", "section": "B.5", "page": 12, "text": "Requirements regarding pricing, bidding and ordering. Prices from the awarded bidder shall remain in effect for the term of the contract.", "criterionIds": ["PRICING"] }, - { "id": "R-E1-EEO", "kind": "certification_form", "section": "E.1", "page": 32, "text": "Equal Opportunity provisions.", "criterionIds": [], "externalFormSchema": true }, - { "id": "R-E3-CLEANAIR", "kind": "certification_form", "section": "E.3", "page": 33, "text": "Clean Air and Water Certification.", "criterionIds": [], "externalFormSchema": true } + { + "id": "R-A1", + "kind": "mandatory_gate", + "section": "A.1", + "page": 4, + "text": "Cover Page completed and signed.", + "criterionIds": [] + }, + { + "id": "R-A2", + "kind": "mandatory_gate", + "section": "A.2", + "page": 5, + "text": "Bidder Checklist completed.", + "criterionIds": [] + }, + { + "id": "R-B2-LOCAL", + "kind": "mandatory_gate", + "section": "B.2", + "page": 5, + "text": "Certification of percentage of local foods to be provided.", + "criterionIds": [ + "GEO_PREF" + ], + "knownMissId": "KM-2" + }, + { + "id": "R-B6C", + "kind": "mandatory_gate", + "section": "B.6(c)", + "page": 5, + "text": "Two bid copies submitted.", + "criterionIds": [], + "determinableFromResponseDocument": false, + "$comment": "A physical submission fact, not a document-content fact. No extraction of any response can establish it, and no better parsing would help. Kept as a mandatory gate because the SFA does enforce it — but it must be satisfied out of band. The schema has no concept for this today; see the note raised on #10." + }, + { + "id": "R-B6D", + "kind": "mandatory_gate", + "section": "B.6(d)", + "page": 5, + "text": "A copy of a current state or local health certificate assuring that all regulations for the preparation of food and the facilities are being met.", + "criterionIds": [ + "EXP_WAREHOUSE" + ] + }, + { + "id": "R-B8-QUEST", + "kind": "mandatory_gate", + "section": "B.8", + "page": 5, + "text": "Distributor Pre-Qualification Questionnaire completed.", + "criterionIds": [ + "EXP_KNOWLEDGE" + ] + }, + { + "id": "R-B8-LICENSE", + "kind": "mandatory_gate", + "section": "B.8", + "page": 5, + "text": "Business license.", + "criterionIds": [] + }, + { + "id": "R-B8-OCCUPANCY", + "kind": "mandatory_gate", + "section": "B.8", + "page": 5, + "text": "Certificate of Occupancy.", + "criterionIds": [] + }, + { + "id": "R-B8-FINANCIALS", + "kind": "mandatory_gate", + "section": "B.8", + "page": 5, + "text": "Two years of audited financial statements.", + "criterionIds": [ + "ORG_CAPACITY" + ] + }, + { + "id": "R-C1-VALUES", + "kind": "mandatory_gate", + "section": "C.1", + "page": 5, + "text": "Response to values statement.", + "criterionIds": [ + "COMMUNITY" + ], + "knownMissId": "KM-3" + }, + { + "id": "R-C5-INSPECTIONS", + "kind": "mandatory_gate", + "section": "C.5", + "page": 5, + "text": "Two most recent health inspections.", + "criterionIds": [ + "EXP_WAREHOUSE" + ] + }, + { + "id": "R-D6-INSURANCE", + "kind": "mandatory_gate", + "section": "D.6", + "page": 5, + "text": "Insurance amounts.", + "criterionIds": [] + }, + { + "id": "R-SCHED-C", + "kind": "pricing", + "section": "Schedule C", + "page": 41, + "text": "Pricing Sheet — separate attachment to be completed in Excel.", + "criterionIds": [ + "PRICING" + ], + "knownMissId": "KM-1" + }, + { + "id": "R-ATT-B", + "kind": "certification_form", + "section": "Attachment B", + "page": 44, + "text": "Certificate Regarding Debarment, Suspension, Ineligibility and Voluntary Exclusion.", + "criterionIds": [], + "externalFormSchema": true + }, + { + "id": "R-ATT-C", + "kind": "certification_form", + "section": "Attachment C", + "page": 46, + "text": "Certificate of Independent Price Determination.", + "criterionIds": [], + "externalFormSchema": true + }, + { + "id": "R-ATT-D", + "kind": "certification_form", + "section": "Attachment D", + "page": 47, + "text": "Permanent Certification Regarding Lobbying.", + "criterionIds": [], + "externalFormSchema": true + }, + { + "id": "R-ATT-A", + "kind": "compliance_schedule", + "section": "Attachment A", + "page": 42, + "text": "Healthy Schools Act requirements compliance.", + "criterionIds": [ + "EXP_KNOWLEDGE" + ] + }, + { + "id": "R-B7-FORMAT", + "kind": "mandatory_gate", + "section": "B.7", + "page": 15, + "text": "The submitted proposal must have four sections, titled: Executive Summary; Pricing; Method of Approach and Implementation Plan; Bidder's Experience, Expertise and Reliability.", + "criterionIds": [] + }, + { + "id": "R-C5-FOODSAFETY", + "kind": "narrative", + "section": "C.5", + "page": 23, + "text": "Warehouse and distribution food safety programme.", + "criterionIds": [ + "EXP_WAREHOUSE", + "FOOD_PACKAGING" + ] + }, + { + "id": "R-C6-PEST", + "kind": "narrative", + "section": "C.6", + "page": 24, + "text": "Stored products pest management programme.", + "criterionIds": [ + "EXP_WAREHOUSE" + ] + }, + { + "id": "R-C7-QA", + "kind": "narrative", + "section": "C.7", + "page": 24, + "text": "Distributor's responsibility to provide quality assurance.", + "criterionIds": [ + "FOOD_PACKAGING" + ] + }, + { + "id": "R-C8-FARM2SCHOOL", + "kind": "narrative", + "section": "C.8", + "page": 25, + "text": "Farm to School participation.", + "criterionIds": [ + "GEO_PREF", + "COMMUNITY" + ] + }, + { + "id": "R-C4-MENUS", + "kind": "narrative", + "section": "C.4", + "page": 23, + "text": "Use of advisory group and menus.", + "criterionIds": [ + "EXP_SERVICE" + ] + }, + { + "id": "R-D1-DELIVERY", + "kind": "narrative", + "section": "D.1", + "page": 26, + "text": "Delivery requirements, including delivery windows per school site.", + "criterionIds": [ + "EXP_DEPENDABILITY" + ] + }, + { + "id": "R-D3-RECORDS", + "kind": "narrative", + "section": "D.3", + "page": 27, + "text": "Record-keeping obligations.", + "criterionIds": [ + "EXP_KNOWLEDGE" + ] + }, + { + "id": "R-USDA-FOODS", + "kind": "narrative", + "section": "B.2/C.3", + "page": 7, + "text": "Utilization of USDA Foods in the product offering, including processing and diversion.", + "criterionIds": [ + "USDA_FOODS" + ] + }, + { + "id": "R-CN-LABELS", + "kind": "narrative", + "section": "B.7", + "page": 15, + "text": "How the Distributor will make nutritional information such as CN labels available to the SFA.", + "criterionIds": [ + "FOOD_PACKAGING", + "EXP_KNOWLEDGE" + ] + }, + { + "id": "R-D8-EPA", + "kind": "narrative", + "section": "D.8", + "page": 30, + "text": "Economic price adjustment and price renegotiation terms.", + "criterionIds": [ + "PRICING" + ] + }, + { + "id": "R-B5-PRICING-TERMS", + "kind": "narrative", + "section": "B.5", + "page": 12, + "text": "Requirements regarding pricing, bidding and ordering. Prices from the awarded bidder shall remain in effect for the term of the contract.", + "criterionIds": [ + "PRICING" + ] + }, + { + "id": "R-E1-EEO", + "kind": "certification_form", + "section": "E.1", + "page": 32, + "text": "Equal Opportunity provisions.", + "criterionIds": [], + "externalFormSchema": true + }, + { + "id": "R-E3-CLEANAIR", + "kind": "certification_form", + "section": "E.3", + "page": 33, + "text": "Clean Air and Water Certification.", + "criterionIds": [], + "externalFormSchema": true + } ], - "knownMisses": [ { "id": "KM-1", @@ -165,7 +544,6 @@ "exercisesSweepBehaviour": "the sweep's second job — running over spans covered ONLY by informational requirements" } ], - "expectedExtractionDifficulty": { "rubricHasNoCriteriaHeading": { "$comment": "The rubric lives under 'SECTION K: Evaluation Factors' → 'K.2 Technical Rating Scale'. Searching for 'evaluation criteria' or 'scoring' returns nothing. Weights are bullet suffixes ('– 40 points'), not a table.", @@ -178,5 +556,11 @@ "textLayerIsClean": { "$comment": "No page required OCR. This document does not exercise the OCR ladder — use SLC Airport for that." } + }, + "requirementNotes": { + "gatesNotDeterminableFromResponse": [ + "R-B6C" + ], + "$comment": "Surfaced by validate-responses.py rather than by design. A mandatory gate whose satisfaction is invisible to the response document is a real category: coverage over it is neither answered nor not_answered, it is unknowable from the artifact. Treating it as not_answered would disqualify two of three bidders for a packaging detail." } } diff --git a/corpus/validate-responses.py b/corpus/validate-responses.py new file mode 100644 index 0000000..ef9eb7c --- /dev/null +++ b/corpus/validate-responses.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Cross-check response truth files against their RFP truth file. + +Run: python3 corpus/validate-responses.py corpus/friendship-pcs +Hand-authored ground truth — nothing else checks these. +""" +import json, sys, glob, os + +root = sys.argv[1] if len(sys.argv) > 1 else 'friendship-pcs' +rfp = json.load(open(os.path.join(root, 'rfp.truth.json'))) +reqs = {r['id']: r for r in rfp['requirements']} +gates = {r['id'] for r in rfp['requirements'] if r['kind'] == 'mandatory_gate'} +# A gate whose satisfaction is invisible to the response document cannot be scored +# from it. Excluded from the silent-gate check; see requirementNotes in the RFP truth. +undeterminable = {r['id'] for r in rfp['requirements'] + if r.get('determinableFromResponseDocument') is False} +COVERAGE = {'answered', 'not_answered', 'indeterminate'} + +errs, warns = [], [] +print(f"RFP: {len(reqs)} requirements, {len(gates)} gates\n") + +for path in sorted(glob.glob(os.path.join(root, 'responses', '*.truth.json'))): + t = json.load(open(path)) + v = t['vendor']['id'] + answered = {} + for a in t['answers']: + rid = a['requirementId'] + if rid not in reqs: + errs.append(f"{v}: answer -> unknown requirement {rid}") + continue + if a['coverage'] not in COVERAGE: + errs.append(f"{v}: {rid} bad coverage '{a['coverage']}'") + if rid in answered: + errs.append(f"{v}: duplicate answer for {rid}") + answered[rid] = a + + missing = sorted(set(reqs) - set(answered)) + if missing: + warns.append(f"{v}: {len(missing)} requirements have no answer entry: {', '.join(missing[:4])}{'...' if len(missing) > 4 else ''}") + + # gate failures must agree with the declared outcome + declared = set(t['expectedOutcome'].get('failedGates', [])) + actual = {r for r, a in answered.items() + if r in gates and (a.get('isGateFailure') or a['coverage'] == 'not_answered')} + flagged = {r for r, a in answered.items() if a.get('isGateFailure')} + if not declared <= flagged: + errs.append(f"{v}: declared failedGates {sorted(declared - flagged)} not marked isGateFailure on the answer") + if t['expectedOutcome']['gatesPassed'] and declared: + errs.append(f"{v}: gatesPassed=true but failedGates is non-empty") + if not t['expectedOutcome']['gatesPassed'] and not declared: + errs.append(f"{v}: gatesPassed=false but no failedGates declared") + + # unflagged gate misses are a real signal, not necessarily an error + silent = actual - declared - set(t['expectedOutcome'].get('atRiskGates', [])) - undeterminable + if silent: + warns.append(f"{v}: gate(s) not_answered but not declared failed or at-risk: {sorted(silent)}") + + for e in t.get('exceptions', []): + rid = e.get('requirementId') + if rid is not None and rid not in reqs: + errs.append(f"{v}: exception {e['id']} -> unknown requirement {rid}") + + counts = {} + for a in answered.values(): + counts[a['coverage']] = counts.get(a['coverage'], 0) + 1 + exc = len(t.get('exceptions', [])) + labelled = sum(1 for e in t.get('exceptions', []) if e.get('labelled')) + print(f" {v:11} binding={t['bindingMechanism']['level']:34} " + f"answered={counts.get('answered',0):2} indet={counts.get('indeterminate',0)} " + f"absent={counts.get('not_answered',0)} exceptions={exc} ({labelled} labelled) " + f"gates={'PASS' if t['expectedOutcome']['gatesPassed'] else 'FAIL'}") + +# corpus-level coverage of the stress-case checklist +files = [json.load(open(p)) for p in glob.glob(os.path.join(root, 'responses', '*.truth.json'))] +checks = { + 'tabbed compliance matrix': any(f['bindingMechanism']['level'] == 'tabbed_restatement' for f in files), + 'free prose, no anchors': any(f['bindingMechanism']['level'] == 'free_prose' for f in files), + 'rate function, not scalar': len({f['priceQuote']['rateStructure']['kind'] for f in files}) >= 3, + 'partially non-responsive': any(not f['expectedOutcome']['gatesPassed'] for f in files), + 'exceptions with price deltas': any(e.get('priceDeltaIfDeclined') for f in files for e in f.get('exceptions', [])), + 'unlabelled exception': any(e.get('labelled') is False for f in files for e in f.get('exceptions', [])), + 'labelled known-misses in RFP': len(rfp.get('knownMisses', [])) >= 3, + 'miskinded-as-informational': any(k['type'] == 'miskinded_as_informational' for k in rfp['knownMisses']), + 'answered-but-negative': any('negative' in str(a.get('$comment','')).lower() for f in files for a in f['answers']), +} +print("\n stress-case checklist") +for k, ok in checks.items(): + print(f" {'x' if ok else ' '} {k}") + if not ok: warns.append(f"stress case not covered: {k}") + +for w in warns: print(f"\n WARN {w}") +for e in errs: print(f"\n ERR {e}") +print("\nVALID" if not errs else "\nINVALID") +sys.exit(1 if errs else 0)