Add three synthesized Friendship PCS responses with ground truth

Covers all nine stress cases from #25, #15 and #16. The lowest bid is
disqualified on two missing health documents and the highest bid is
strongest on the SFA's own geographic-preference criterion, so a ranking
that cannot represent absence gets the field exactly backwards.

Deliberate traps: a planted $400 arithmetic error in a stated total, an
unverifiable total whose unit price lives only in the external Excel, an
unlabelled exception, a non-priceable exception that is really a conditional
withdrawal, and an answered-but-negative response.

Second schema finding, surfaced by the validator: R-B6C is a mandatory gate
that no response document can establish. Coverage over it is unknowable
rather than absent.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Christian
2026-08-03 18:39:08 -04:00
parent e81da40947
commit 19e7ffe92a
9 changed files with 1509 additions and 71 deletions

View File

@@ -6,7 +6,6 @@
"",
"Requirement ids are stable and are what response truth files answer against."
],
"solicitation": {
"title": "Request for Proposal for the Supply and Delivery of Food Items",
"issuer": "Friendship Public Charter School",
@@ -14,10 +13,16 @@
"issueDate": "2021-03-15",
"closingDate": "2021-04-16",
"pages": 52,
"programContext": ["NSLP", "SBP", "ASSP", "FFVP", "CACFP", "SFSP"],
"programContext": [
"NSLP",
"SBP",
"ASSP",
"FFVP",
"CACFP",
"SFSP"
],
"siteCount": 8
},
"bindingMechanism": {
"level": "named_section",
"$comment": [
@@ -27,42 +32,151 @@
"This is the single most important fact about scoring this document."
],
"requiredSections": [
{ "ordinal": 1, "title": "Executive Summary", "mapsToCriterion": null },
{ "ordinal": 2, "title": "Pricing", "mapsToCriterion": "PRICING" },
{ "ordinal": 3, "title": "Method of Approach and Implementation Plan", "mapsToCriterion": "METHOD" },
{ "ordinal": 4, "title": "Bidder's Experience, Expertise and Reliability", "mapsToCriterion": "EXPERIENCE" }
{
"ordinal": 1,
"title": "Executive Summary",
"mapsToCriterion": null
},
{
"ordinal": 2,
"title": "Pricing",
"mapsToCriterion": "PRICING"
},
{
"ordinal": 3,
"title": "Method of Approach and Implementation Plan",
"mapsToCriterion": "METHOD"
},
{
"ordinal": 4,
"title": "Bidder's Experience, Expertise and Reliability",
"mapsToCriterion": "EXPERIENCE"
}
]
},
"ratingScale": {
"$comment": "Per-document, never normalised (#10, #22). The prose anchor is load-bearing, not the integer.",
"min": 0,
"max": 5,
"anchors": [
{ "points": 0, "label": "Completely Deficient", "anchor": "Elements is missing or is completely nonresponsive to the request" },
{ "points": 1, "label": "Unsatisfactory", "anchor": "Fails to meet minimum requirements; major deficiencies which are not correctable" },
{ "points": 2, "label": "Below Average", "anchor": "Marginally meets minimum requirements; significant deficiencies which may be correctable" },
{ "points": 3, "label": "Average", "anchor": "Meets requirements; only minor deficiencies which are correctable" },
{ "points": 4, "label": "Very Good", "anchor": "Meets requirements; no deficiencies" },
{ "points": 5, "label": "Exceptional", "anchor": "Exceeds most, if not all requirements; no deficiencies" }
{
"points": 0,
"label": "Completely Deficient",
"anchor": "Elements is missing or is completely nonresponsive to the request"
},
{
"points": 1,
"label": "Unsatisfactory",
"anchor": "Fails to meet minimum requirements; major deficiencies which are not correctable"
},
{
"points": 2,
"label": "Below Average",
"anchor": "Marginally meets minimum requirements; significant deficiencies which may be correctable"
},
{
"points": 3,
"label": "Average",
"anchor": "Meets requirements; only minor deficiencies which are correctable"
},
{
"points": 4,
"label": "Very Good",
"anchor": "Meets requirements; no deficiencies"
},
{
"points": 5,
"label": "Exceptional",
"anchor": "Exceeds most, if not all requirements; no deficiencies"
}
]
},
"criteria": [
{ "id": "PRICING", "parentId": null, "label": "Pricing", "maxPoints": 40, "page": 37 },
{ "id": "METHOD", "parentId": null, "label": "Method of Approach and Implementation", "maxPoints": 30, "page": 37 },
{ "id": "FOOD_PACKAGING", "parentId": "METHOD", "label": "Food and Packaging (Appearance, Quality, Taste and Acceptance)", "maxPoints": 15, "page": 37 },
{ "id": "USDA_FOODS", "parentId": "METHOD", "label": "Utilization of USDA Foods", "maxPoints": 10, "page": 37 },
{ "id": "GEO_PREF", "parentId": "METHOD", "label": "Geographic preference — locally grown or raised foods", "maxPoints": 5, "page": 37 },
{ "id": "EXPERIENCE", "parentId": null, "label": "Bidder's Experience, Expertise, & Reliability", "maxPoints": 20, "page": 37 },
{ "id": "EXP_KNOWLEDGE", "parentId": "EXPERIENCE", "label": "Knowledge of USDA, District requirements", "maxPoints": 5, "page": 37 },
{ "id": "EXP_DEPENDABILITY", "parentId": "EXPERIENCE", "label": "Dependability (on-time deliveries, accuracy filling orders)", "maxPoints": 5, "page": 37 },
{ "id": "EXP_WAREHOUSE", "parentId": "EXPERIENCE", "label": "Inspection of warehouse and other facilities", "maxPoints": 5, "page": 37 },
{ "id": "EXP_SERVICE", "parentId": "EXPERIENCE", "label": "Customer Service", "maxPoints": 5, "page": 37 },
{ "id": "ORG_CAPACITY", "parentId": null, "label": "Proven Organizational Capacity", "maxPoints": 5, "page": 38 },
{ "id": "COMMUNITY", "parentId": null, "label": "Community Connection, Values Alignment", "maxPoints": 5, "page": 38 }
{
"id": "PRICING",
"parentId": null,
"label": "Pricing",
"maxPoints": 40,
"page": 37
},
{
"id": "METHOD",
"parentId": null,
"label": "Method of Approach and Implementation",
"maxPoints": 30,
"page": 37
},
{
"id": "FOOD_PACKAGING",
"parentId": "METHOD",
"label": "Food and Packaging (Appearance, Quality, Taste and Acceptance)",
"maxPoints": 15,
"page": 37
},
{
"id": "USDA_FOODS",
"parentId": "METHOD",
"label": "Utilization of USDA Foods",
"maxPoints": 10,
"page": 37
},
{
"id": "GEO_PREF",
"parentId": "METHOD",
"label": "Geographic preference — locally grown or raised foods",
"maxPoints": 5,
"page": 37
},
{
"id": "EXPERIENCE",
"parentId": null,
"label": "Bidder's Experience, Expertise, & Reliability",
"maxPoints": 20,
"page": 37
},
{
"id": "EXP_KNOWLEDGE",
"parentId": "EXPERIENCE",
"label": "Knowledge of USDA, District requirements",
"maxPoints": 5,
"page": 37
},
{
"id": "EXP_DEPENDABILITY",
"parentId": "EXPERIENCE",
"label": "Dependability (on-time deliveries, accuracy filling orders)",
"maxPoints": 5,
"page": 37
},
{
"id": "EXP_WAREHOUSE",
"parentId": "EXPERIENCE",
"label": "Inspection of warehouse and other facilities",
"maxPoints": 5,
"page": 37
},
{
"id": "EXP_SERVICE",
"parentId": "EXPERIENCE",
"label": "Customer Service",
"maxPoints": 5,
"page": 37
},
{
"id": "ORG_CAPACITY",
"parentId": null,
"label": "Proven Organizational Capacity",
"maxPoints": 5,
"page": 38
},
{
"id": "COMMUNITY",
"parentId": null,
"label": "Community Connection, Values Alignment",
"maxPoints": 5,
"page": 38
}
],
"criteriaNotes": {
"totalPoints": 100,
"nestingIsInconsistent": {
@@ -73,8 +187,14 @@
"they are requirements mapped to the criterion, not sub-criteria. Identical visual",
"nesting, two different meanings, in one document."
],
"weightedParents": ["METHOD", "EXPERIENCE"],
"guidanceOnlyParents": ["ORG_CAPACITY", "COMMUNITY"]
"weightedParents": [
"METHOD",
"EXPERIENCE"
],
"guidanceOnlyParents": [
"ORG_CAPACITY",
"COMMUNITY"
]
},
"scoringLevelIsAmbiguous": {
"$comment": [
@@ -86,48 +206,307 @@
]
}
},
"requirements": [
{ "id": "R-A1", "kind": "mandatory_gate", "section": "A.1", "page": 4, "text": "Cover Page completed and signed.", "criterionIds": [] },
{ "id": "R-A2", "kind": "mandatory_gate", "section": "A.2", "page": 5, "text": "Bidder Checklist completed.", "criterionIds": [] },
{ "id": "R-B2-LOCAL", "kind": "mandatory_gate", "section": "B.2", "page": 5, "text": "Certification of percentage of local foods to be provided.", "criterionIds": ["GEO_PREF"], "knownMissId": "KM-2" },
{ "id": "R-B6C", "kind": "mandatory_gate", "section": "B.6(c)", "page": 5, "text": "Two bid copies submitted.", "criterionIds": [] },
{ "id": "R-B6D", "kind": "mandatory_gate", "section": "B.6(d)", "page": 5, "text": "A copy of a current state or local health certificate assuring that all regulations for the preparation of food and the facilities are being met.", "criterionIds": ["EXP_WAREHOUSE"] },
{ "id": "R-B8-QUEST", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Distributor Pre-Qualification Questionnaire completed.", "criterionIds": ["EXP_KNOWLEDGE"] },
{ "id": "R-B8-LICENSE", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Business license.", "criterionIds": [] },
{ "id": "R-B8-OCCUPANCY", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Certificate of Occupancy.", "criterionIds": [] },
{ "id": "R-B8-FINANCIALS", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Two years of audited financial statements.", "criterionIds": ["ORG_CAPACITY"] },
{ "id": "R-C1-VALUES", "kind": "mandatory_gate", "section": "C.1", "page": 5, "text": "Response to values statement.", "criterionIds": ["COMMUNITY"], "knownMissId": "KM-3" },
{ "id": "R-C5-INSPECTIONS", "kind": "mandatory_gate", "section": "C.5", "page": 5, "text": "Two most recent health inspections.", "criterionIds": ["EXP_WAREHOUSE"] },
{ "id": "R-D6-INSURANCE", "kind": "mandatory_gate", "section": "D.6", "page": 5, "text": "Insurance amounts.", "criterionIds": [] },
{ "id": "R-SCHED-C", "kind": "pricing", "section": "Schedule C", "page": 41, "text": "Pricing Sheet — separate attachment to be completed in Excel.", "criterionIds": ["PRICING"], "knownMissId": "KM-1" },
{ "id": "R-ATT-B", "kind": "certification_form", "section": "Attachment B", "page": 44, "text": "Certificate Regarding Debarment, Suspension, Ineligibility and Voluntary Exclusion.", "criterionIds": [], "externalFormSchema": true },
{ "id": "R-ATT-C", "kind": "certification_form", "section": "Attachment C", "page": 46, "text": "Certificate of Independent Price Determination.", "criterionIds": [], "externalFormSchema": true },
{ "id": "R-ATT-D", "kind": "certification_form", "section": "Attachment D", "page": 47, "text": "Permanent Certification Regarding Lobbying.", "criterionIds": [], "externalFormSchema": true },
{ "id": "R-ATT-A", "kind": "compliance_schedule", "section": "Attachment A", "page": 42, "text": "Healthy Schools Act requirements compliance.", "criterionIds": ["EXP_KNOWLEDGE"] },
{ "id": "R-B7-FORMAT", "kind": "mandatory_gate", "section": "B.7", "page": 15, "text": "The submitted proposal must have four sections, titled: Executive Summary; Pricing; Method of Approach and Implementation Plan; Bidder's Experience, Expertise and Reliability.", "criterionIds": [] },
{ "id": "R-C5-FOODSAFETY", "kind": "narrative", "section": "C.5", "page": 23, "text": "Warehouse and distribution food safety programme.", "criterionIds": ["EXP_WAREHOUSE", "FOOD_PACKAGING"] },
{ "id": "R-C6-PEST", "kind": "narrative", "section": "C.6", "page": 24, "text": "Stored products pest management programme.", "criterionIds": ["EXP_WAREHOUSE"] },
{ "id": "R-C7-QA", "kind": "narrative", "section": "C.7", "page": 24, "text": "Distributor's responsibility to provide quality assurance.", "criterionIds": ["FOOD_PACKAGING"] },
{ "id": "R-C8-FARM2SCHOOL", "kind": "narrative", "section": "C.8", "page": 25, "text": "Farm to School participation.", "criterionIds": ["GEO_PREF", "COMMUNITY"] },
{ "id": "R-C4-MENUS", "kind": "narrative", "section": "C.4", "page": 23, "text": "Use of advisory group and menus.", "criterionIds": ["EXP_SERVICE"] },
{ "id": "R-D1-DELIVERY", "kind": "narrative", "section": "D.1", "page": 26, "text": "Delivery requirements, including delivery windows per school site.", "criterionIds": ["EXP_DEPENDABILITY"] },
{ "id": "R-D3-RECORDS", "kind": "narrative", "section": "D.3", "page": 27, "text": "Record-keeping obligations.", "criterionIds": ["EXP_KNOWLEDGE"] },
{ "id": "R-USDA-FOODS", "kind": "narrative", "section": "B.2/C.3", "page": 7, "text": "Utilization of USDA Foods in the product offering, including processing and diversion.", "criterionIds": ["USDA_FOODS"] },
{ "id": "R-CN-LABELS", "kind": "narrative", "section": "B.7", "page": 15, "text": "How the Distributor will make nutritional information such as CN labels available to the SFA.", "criterionIds": ["FOOD_PACKAGING", "EXP_KNOWLEDGE"] },
{ "id": "R-D8-EPA", "kind": "narrative", "section": "D.8", "page": 30, "text": "Economic price adjustment and price renegotiation terms.", "criterionIds": ["PRICING"] },
{ "id": "R-B5-PRICING-TERMS", "kind": "narrative", "section": "B.5", "page": 12, "text": "Requirements regarding pricing, bidding and ordering. Prices from the awarded bidder shall remain in effect for the term of the contract.", "criterionIds": ["PRICING"] },
{ "id": "R-E1-EEO", "kind": "certification_form", "section": "E.1", "page": 32, "text": "Equal Opportunity provisions.", "criterionIds": [], "externalFormSchema": true },
{ "id": "R-E3-CLEANAIR", "kind": "certification_form", "section": "E.3", "page": 33, "text": "Clean Air and Water Certification.", "criterionIds": [], "externalFormSchema": true }
{
"id": "R-A1",
"kind": "mandatory_gate",
"section": "A.1",
"page": 4,
"text": "Cover Page completed and signed.",
"criterionIds": []
},
{
"id": "R-A2",
"kind": "mandatory_gate",
"section": "A.2",
"page": 5,
"text": "Bidder Checklist completed.",
"criterionIds": []
},
{
"id": "R-B2-LOCAL",
"kind": "mandatory_gate",
"section": "B.2",
"page": 5,
"text": "Certification of percentage of local foods to be provided.",
"criterionIds": [
"GEO_PREF"
],
"knownMissId": "KM-2"
},
{
"id": "R-B6C",
"kind": "mandatory_gate",
"section": "B.6(c)",
"page": 5,
"text": "Two bid copies submitted.",
"criterionIds": [],
"determinableFromResponseDocument": false,
"$comment": "A physical submission fact, not a document-content fact. No extraction of any response can establish it, and no better parsing would help. Kept as a mandatory gate because the SFA does enforce it — but it must be satisfied out of band. The schema has no concept for this today; see the note raised on #10."
},
{
"id": "R-B6D",
"kind": "mandatory_gate",
"section": "B.6(d)",
"page": 5,
"text": "A copy of a current state or local health certificate assuring that all regulations for the preparation of food and the facilities are being met.",
"criterionIds": [
"EXP_WAREHOUSE"
]
},
{
"id": "R-B8-QUEST",
"kind": "mandatory_gate",
"section": "B.8",
"page": 5,
"text": "Distributor Pre-Qualification Questionnaire completed.",
"criterionIds": [
"EXP_KNOWLEDGE"
]
},
{
"id": "R-B8-LICENSE",
"kind": "mandatory_gate",
"section": "B.8",
"page": 5,
"text": "Business license.",
"criterionIds": []
},
{
"id": "R-B8-OCCUPANCY",
"kind": "mandatory_gate",
"section": "B.8",
"page": 5,
"text": "Certificate of Occupancy.",
"criterionIds": []
},
{
"id": "R-B8-FINANCIALS",
"kind": "mandatory_gate",
"section": "B.8",
"page": 5,
"text": "Two years of audited financial statements.",
"criterionIds": [
"ORG_CAPACITY"
]
},
{
"id": "R-C1-VALUES",
"kind": "mandatory_gate",
"section": "C.1",
"page": 5,
"text": "Response to values statement.",
"criterionIds": [
"COMMUNITY"
],
"knownMissId": "KM-3"
},
{
"id": "R-C5-INSPECTIONS",
"kind": "mandatory_gate",
"section": "C.5",
"page": 5,
"text": "Two most recent health inspections.",
"criterionIds": [
"EXP_WAREHOUSE"
]
},
{
"id": "R-D6-INSURANCE",
"kind": "mandatory_gate",
"section": "D.6",
"page": 5,
"text": "Insurance amounts.",
"criterionIds": []
},
{
"id": "R-SCHED-C",
"kind": "pricing",
"section": "Schedule C",
"page": 41,
"text": "Pricing Sheet — separate attachment to be completed in Excel.",
"criterionIds": [
"PRICING"
],
"knownMissId": "KM-1"
},
{
"id": "R-ATT-B",
"kind": "certification_form",
"section": "Attachment B",
"page": 44,
"text": "Certificate Regarding Debarment, Suspension, Ineligibility and Voluntary Exclusion.",
"criterionIds": [],
"externalFormSchema": true
},
{
"id": "R-ATT-C",
"kind": "certification_form",
"section": "Attachment C",
"page": 46,
"text": "Certificate of Independent Price Determination.",
"criterionIds": [],
"externalFormSchema": true
},
{
"id": "R-ATT-D",
"kind": "certification_form",
"section": "Attachment D",
"page": 47,
"text": "Permanent Certification Regarding Lobbying.",
"criterionIds": [],
"externalFormSchema": true
},
{
"id": "R-ATT-A",
"kind": "compliance_schedule",
"section": "Attachment A",
"page": 42,
"text": "Healthy Schools Act requirements compliance.",
"criterionIds": [
"EXP_KNOWLEDGE"
]
},
{
"id": "R-B7-FORMAT",
"kind": "mandatory_gate",
"section": "B.7",
"page": 15,
"text": "The submitted proposal must have four sections, titled: Executive Summary; Pricing; Method of Approach and Implementation Plan; Bidder's Experience, Expertise and Reliability.",
"criterionIds": []
},
{
"id": "R-C5-FOODSAFETY",
"kind": "narrative",
"section": "C.5",
"page": 23,
"text": "Warehouse and distribution food safety programme.",
"criterionIds": [
"EXP_WAREHOUSE",
"FOOD_PACKAGING"
]
},
{
"id": "R-C6-PEST",
"kind": "narrative",
"section": "C.6",
"page": 24,
"text": "Stored products pest management programme.",
"criterionIds": [
"EXP_WAREHOUSE"
]
},
{
"id": "R-C7-QA",
"kind": "narrative",
"section": "C.7",
"page": 24,
"text": "Distributor's responsibility to provide quality assurance.",
"criterionIds": [
"FOOD_PACKAGING"
]
},
{
"id": "R-C8-FARM2SCHOOL",
"kind": "narrative",
"section": "C.8",
"page": 25,
"text": "Farm to School participation.",
"criterionIds": [
"GEO_PREF",
"COMMUNITY"
]
},
{
"id": "R-C4-MENUS",
"kind": "narrative",
"section": "C.4",
"page": 23,
"text": "Use of advisory group and menus.",
"criterionIds": [
"EXP_SERVICE"
]
},
{
"id": "R-D1-DELIVERY",
"kind": "narrative",
"section": "D.1",
"page": 26,
"text": "Delivery requirements, including delivery windows per school site.",
"criterionIds": [
"EXP_DEPENDABILITY"
]
},
{
"id": "R-D3-RECORDS",
"kind": "narrative",
"section": "D.3",
"page": 27,
"text": "Record-keeping obligations.",
"criterionIds": [
"EXP_KNOWLEDGE"
]
},
{
"id": "R-USDA-FOODS",
"kind": "narrative",
"section": "B.2/C.3",
"page": 7,
"text": "Utilization of USDA Foods in the product offering, including processing and diversion.",
"criterionIds": [
"USDA_FOODS"
]
},
{
"id": "R-CN-LABELS",
"kind": "narrative",
"section": "B.7",
"page": 15,
"text": "How the Distributor will make nutritional information such as CN labels available to the SFA.",
"criterionIds": [
"FOOD_PACKAGING",
"EXP_KNOWLEDGE"
]
},
{
"id": "R-D8-EPA",
"kind": "narrative",
"section": "D.8",
"page": 30,
"text": "Economic price adjustment and price renegotiation terms.",
"criterionIds": [
"PRICING"
]
},
{
"id": "R-B5-PRICING-TERMS",
"kind": "narrative",
"section": "B.5",
"page": 12,
"text": "Requirements regarding pricing, bidding and ordering. Prices from the awarded bidder shall remain in effect for the term of the contract.",
"criterionIds": [
"PRICING"
]
},
{
"id": "R-E1-EEO",
"kind": "certification_form",
"section": "E.1",
"page": 32,
"text": "Equal Opportunity provisions.",
"criterionIds": [],
"externalFormSchema": true
},
{
"id": "R-E3-CLEANAIR",
"kind": "certification_form",
"section": "E.3",
"page": 33,
"text": "Clean Air and Water Certification.",
"criterionIds": [],
"externalFormSchema": true
}
],
"knownMisses": [
{
"id": "KM-1",
@@ -165,7 +544,6 @@
"exercisesSweepBehaviour": "the sweep's second job — running over spans covered ONLY by informational requirements"
}
],
"expectedExtractionDifficulty": {
"rubricHasNoCriteriaHeading": {
"$comment": "The rubric lives under 'SECTION K: Evaluation Factors' → 'K.2 Technical Rating Scale'. Searching for 'evaluation criteria' or 'scoring' returns nothing. Weights are bullet suffixes (' 40 points'), not a table.",
@@ -178,5 +556,11 @@
"textLayerIsClean": {
"$comment": "No page required OCR. This document does not exercise the OCR ladder — use SLC Airport for that."
}
},
"requirementNotes": {
"gatesNotDeterminableFromResponse": [
"R-B6C"
],
"$comment": "Surfaced by validate-responses.py rather than by design. A mandatory gate whose satisfaction is invisible to the response document is a real category: coverage over it is neither answered nor not_answered, it is unknowable from the artifact. Treating it as not_answered would disqualify two of three bidders for a packaging detail."
}
}