Add three synthesized Friendship PCS responses with ground truth
Covers all nine stress cases from #25, #15 and #16. The lowest bid is disqualified on two missing health documents and the highest bid is strongest on the SFA's own geographic-preference criterion, so a ranking that cannot represent absence gets the field exactly backwards. Deliberate traps: a planted $400 arithmetic error in a stated total, an unverifiable total whose unit price lives only in the external Excel, an unlabelled exception, a non-priceable exception that is really a conditional withdrawal, and an answered-but-negative response. Second schema finding, surfaced by the validator: R-B6C is a mandatory gate that no response document can establish. Coverage over it is unknowable rather than absent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Christian
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"$comment": "Ground truth for source/anacostia-farm-collective.md. Mid-gradient binding, and the exceptions case.",
|
||||
"vendor": { "id": "ANACOSTIA", "name": "Anacostia Farm Collective", "location": "Washington, DC (Ward 8)" },
|
||||
|
||||
"bindingMechanism": {
|
||||
"level": "named_section_with_partial_enumeration",
|
||||
"$comment": "#22's middle tier. Four required sections are correctly titled and an enclosures paragraph enumerates artifacts with RFP references inline — but there is no matrix, so gates bind by name while narrative requirements bind only by section."
|
||||
},
|
||||
|
||||
"expectedOutcome": {
|
||||
"gatesPassed": true,
|
||||
"narrative": "All gates satisfied. Highest price at $624,315, or $652,465 if exceptions 1 and 2 are declined. Strongest on geographic preference by a wide margin (61% vs 18%), openly weakest on USDA Foods, and materially below the field on organizational scale. The right award depends entirely on how the SFA weights local content against price — which is the judgement the retailer is supposed to make and the system is not."
|
||||
},
|
||||
|
||||
"answers": [
|
||||
{ "requirementId": "R-A1", "coverage": "answered", "evidence": "Cover block present" },
|
||||
{ "requirementId": "R-A2", "coverage": "indeterminate", "evidence": "Enclosures paragraph enumerates items in checklist order but is not the checklist form itself" },
|
||||
{ "requirementId": "R-B2-LOCAL", "coverage": "answered", "evidence": "61% certified; audit right accepted; certification enclosed under B.2", "value": { "localPercent": 61 } },
|
||||
{ "requirementId": "R-B6C", "coverage": "not_answered", "evidence": null },
|
||||
{ "requirementId": "R-B6D", "coverage": "answered", "evidence": "DC Department of Health food establishment certificate, exp. 2022-06-30" },
|
||||
{ "requirementId": "R-B7-FORMAT", "coverage": "answered", "evidence": "Four sections, titles exact" },
|
||||
{ "requirementId": "R-B8-QUEST", "coverage": "answered", "evidence": "References provided in questionnaire" },
|
||||
{ "requirementId": "R-B8-LICENSE", "coverage": "answered", "evidence": "DC BBL 410203000891" },
|
||||
{ "requirementId": "R-B8-OCCUPANCY", "coverage": "answered", "evidence": "Enclosed" },
|
||||
{ "requirementId": "R-B8-FINANCIALS", "coverage": "answered", "evidence": "FY2019 and FY2020, Mercer Group CPAs" },
|
||||
{ "requirementId": "R-C1-VALUES", "coverage": "answered", "evidence": "Explicit 'Response to values statement (C.1)' subsection in Section Four" },
|
||||
{ "requirementId": "R-C5-INSPECTIONS", "coverage": "answered", "evidence": "2020-09-22 and 2021-01-14, both passed" },
|
||||
{ "requirementId": "R-D6-INSURANCE", "coverage": "answered", "evidence": "Certificate of insurance enclosed",
|
||||
"$comment": "Enclosed but NO AMOUNTS STATED, unlike the other two bidders. The requirement is 'Insurance amounts.' A strict reading makes this indeterminate; a lenient one accepts the certificate. Deliberately left ambiguous — it is the kind of call that should reach a human." },
|
||||
{ "requirementId": "R-SCHED-C", "coverage": "answered", "evidence": "Submitted on Schedule C in the Excel workbook" },
|
||||
{ "requirementId": "R-ATT-A", "coverage": "answered", "evidence": "Supplies four DC charter SFAs under HSA; certification designed to support the local sourcing subsidy claim" },
|
||||
{ "requirementId": "R-ATT-B", "coverage": "answered", "evidence": "Signed" },
|
||||
{ "requirementId": "R-ATT-C", "coverage": "answered", "evidence": "Signed" },
|
||||
{ "requirementId": "R-ATT-D", "coverage": "answered", "evidence": "Signed" },
|
||||
{ "requirementId": "R-E1-EEO", "coverage": "answered", "evidence": "Signed" },
|
||||
{ "requirementId": "R-E3-CLEANAIR", "coverage": "answered", "evidence": "Signed" },
|
||||
|
||||
{ "requirementId": "R-USDA-FOODS", "coverage": "answered",
|
||||
"evidence": "'This is our weakest area and we will not overstate it. We are not currently an approved USDA Foods processor.'",
|
||||
"$comment": "ANSWERED, and answered in the NEGATIVE. The requirement is squarely addressed; the substance is a disclosure of inability. A system that treats 'answered' as 'satisfied' scores this identically to Potomac's 340,000 lbs. It must not." },
|
||||
{ "requirementId": "R-C8-FARM2SCHOOL", "coverage": "answered", "evidence": "34 member farms in the Chesapeake foodshed; two within the District" },
|
||||
{ "requirementId": "R-C7-QA", "coverage": "answered", "evidence": "Farm-level GAP certification, annual audit, credit at receipt" },
|
||||
{ "requirementId": "R-C5-FOODSAFETY", "coverage": "answered", "evidence": "DC DoH certified facility; inspections enclosed" },
|
||||
{ "requirementId": "R-C6-PEST", "coverage": "answered", "evidence": "Licensed applicator monthly, semi-annual third-party audit" },
|
||||
{ "requirementId": "R-CN-LABELS", "coverage": "answered",
|
||||
"evidence": "'Whole produce does not carry CN labels; for those items we supply the crediting documentation the meal pattern requires.'",
|
||||
"$comment": "A QUALIFIED answer — satisfies the requirement for processed items and explains why it cannot for whole produce. Neither fully answered nor a gap." },
|
||||
{ "requirementId": "R-D1-DELIVERY", "coverage": "answered", "evidence": "Addressed, and subject to Exception 1 on delivery windows",
|
||||
"hasException": "EX-1" },
|
||||
{ "requirementId": "R-B5-PRICING-TERMS", "coverage": "answered", "evidence": "Firm within stated bands", "hasException": "EX-2" },
|
||||
{ "requirementId": "R-D8-EPA", "coverage": "answered", "evidence": "Renegotiation of the out-of-region band, per Exception 2", "hasException": "EX-2" },
|
||||
{ "requirementId": "R-C4-MENUS", "coverage": "not_answered", "evidence": null },
|
||||
{ "requirementId": "R-D3-RECORDS", "coverage": "not_answered", "evidence": null }
|
||||
],
|
||||
|
||||
"priceQuote": {
|
||||
"rateStructure": {
|
||||
"kind": "seasonal_piecewise",
|
||||
"$comment": "The hardest of the three, and the one most likely to break a naive model. Cost depends on WHEN volume falls, not only how much — so derived_cost cannot be computed without the school calendar. #22 found exactly this shape in the wild (SLC rent varying by contract year and sales threshold).",
|
||||
"unit": "case",
|
||||
"baseProduceCasePrice": 31.20,
|
||||
"appliesTo": "produce only; grocery, dairy and protein are flat on Schedule C",
|
||||
"bands": [
|
||||
{ "id": "peak_regional", "months": [6, 7, 8, 9, 10], "multiplier": 0.88 },
|
||||
{ "id": "shoulder", "months": [4, 5, 11], "multiplier": 1.00 },
|
||||
{ "id": "out_of_region", "months": [12, 1, 2, 3], "multiplier": 1.19 }
|
||||
],
|
||||
"requiresCalendar": true,
|
||||
"$calendarNote": "The 180-day school calendar falls disproportionately outside the peak band — the vendor says so explicitly. Any cost comparison that ignores the calendar flatters this bid."
|
||||
},
|
||||
"statedBaseYearTotal": 624315,
|
||||
"statedTotalIfExceptionsDeclined": 652465,
|
||||
"$verify": "624315 + 18400 + 9750 = 652465. Internally consistent. Contrast Potomac's planted $400 error."
|
||||
},
|
||||
|
||||
"exceptions": [
|
||||
{
|
||||
"id": "EX-1", "labelled": true, "requirementId": "R-D1-DELIVERY",
|
||||
"text": "Delivery windows beginning no earlier than 6:30 AM at all sites; Schedule A specifies several beginning at 5:30 AM.",
|
||||
"priceDeltaIfDeclined": 18400,
|
||||
"$comment": "A hard operational conflict with Schedule A, not a preference."
|
||||
},
|
||||
{
|
||||
"id": "EX-2", "labelled": true, "requirementId": "R-B5-PRICING-TERMS",
|
||||
"text": "Out-of-region band subject to renegotiation if PPI for fresh vegetables moves more than 12% between award and 1 December.",
|
||||
"priceDeltaIfDeclined": 9750
|
||||
},
|
||||
{
|
||||
"id": "EX-3", "labelled": true, "requirementId": null,
|
||||
"targetSection": "C.3",
|
||||
"text": "Pack configuration agreed during onboarding rather than fixed at award, produce categories only.",
|
||||
"priceDeltaIfDeclined": null,
|
||||
"isBidWithdrawalCondition": true,
|
||||
"$comment": "NOT PRICEABLE. If declined the vendor cannot bid produce at all — a conditional withdrawal expressed as an exception. A price-delta field cannot hold it, and a system that models exceptions purely as monetary adjustments loses the most consequential one of the three. It also targets a requirement the RFP truth file does not enumerate (C.3 Specifications), which is itself a finding."
|
||||
}
|
||||
],
|
||||
|
||||
"extractionDifficulty": {
|
||||
"expectRecall": "medium",
|
||||
"$comment": "Gates bind reliably via the enclosures paragraph. Narrative requirements bind by section only. The exceptions block is explicitly headed and should extract cleanly — it is the control against which Chesapeake's unlabelled exception is measured."
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user