diff --git a/corpus/friendship-pcs/rfp.truth.json b/corpus/friendship-pcs/rfp.truth.json new file mode 100644 index 0000000..7f8d3cf --- /dev/null +++ b/corpus/friendship-pcs/rfp.truth.json @@ -0,0 +1,182 @@ +{ + "$comment": [ + "Ground truth for corpus/friendship-pcs/rfp.pdf — what a correct extraction SHOULD produce.", + "Hand-authored from the source document, not generated. Where this disagrees with an", + "extractor, the extractor is wrong. See NOTES.md for how it was derived.", + "", + "Requirement ids are stable and are what response truth files answer against." + ], + + "solicitation": { + "title": "Request for Proposal for the Supply and Delivery of Food Items", + "issuer": "Friendship Public Charter School", + "issuerLocation": "Washington, DC", + "issueDate": "2021-03-15", + "closingDate": "2021-04-16", + "pages": 52, + "programContext": ["NSLP", "SBP", "ASSP", "FFVP", "CACFP", "SFSP"], + "siteCount": 8 + }, + + "bindingMechanism": { + "level": "named_section", + "$comment": [ + "#22's coarse tier. B.7 mandates exactly four titled sections, which map to rubric", + "CATEGORIES but not to their sub-criteria. The 15/10/5 split inside Method of Approach", + "has no structural anchor in the response — all three are judged from one prose block.", + "This is the single most important fact about scoring this document." + ], + "requiredSections": [ + { "ordinal": 1, "title": "Executive Summary", "mapsToCriterion": null }, + { "ordinal": 2, "title": "Pricing", "mapsToCriterion": "PRICING" }, + { "ordinal": 3, "title": "Method of Approach and Implementation Plan", "mapsToCriterion": "METHOD" }, + { "ordinal": 4, "title": "Bidder's Experience, Expertise and Reliability", "mapsToCriterion": "EXPERIENCE" } + ] + }, + + "ratingScale": { + "$comment": "Per-document, never normalised (#10, #22). The prose anchor is load-bearing, not the integer.", + "min": 0, + "max": 5, + "anchors": [ + { "points": 0, "label": "Completely Deficient", "anchor": "Elements is missing or is completely nonresponsive to the request" }, + { "points": 1, "label": "Unsatisfactory", "anchor": "Fails to meet minimum requirements; major deficiencies which are not correctable" }, + { "points": 2, "label": "Below Average", "anchor": "Marginally meets minimum requirements; significant deficiencies which may be correctable" }, + { "points": 3, "label": "Average", "anchor": "Meets requirements; only minor deficiencies which are correctable" }, + { "points": 4, "label": "Very Good", "anchor": "Meets requirements; no deficiencies" }, + { "points": 5, "label": "Exceptional", "anchor": "Exceeds most, if not all requirements; no deficiencies" } + ] + }, + + "criteria": [ + { "id": "PRICING", "parentId": null, "label": "Pricing", "maxPoints": 40, "page": 37 }, + { "id": "METHOD", "parentId": null, "label": "Method of Approach and Implementation", "maxPoints": 30, "page": 37 }, + { "id": "FOOD_PACKAGING", "parentId": "METHOD", "label": "Food and Packaging (Appearance, Quality, Taste and Acceptance)", "maxPoints": 15, "page": 37 }, + { "id": "USDA_FOODS", "parentId": "METHOD", "label": "Utilization of USDA Foods", "maxPoints": 10, "page": 37 }, + { "id": "GEO_PREF", "parentId": "METHOD", "label": "Geographic preference — locally grown or raised foods", "maxPoints": 5, "page": 37 }, + { "id": "EXPERIENCE", "parentId": null, "label": "Bidder's Experience, Expertise, & Reliability", "maxPoints": 20, "page": 37 }, + { "id": "EXP_KNOWLEDGE", "parentId": "EXPERIENCE", "label": "Knowledge of USDA, District requirements", "maxPoints": 5, "page": 37 }, + { "id": "EXP_DEPENDABILITY", "parentId": "EXPERIENCE", "label": "Dependability (on-time deliveries, accuracy filling orders)", "maxPoints": 5, "page": 37 }, + { "id": "EXP_WAREHOUSE", "parentId": "EXPERIENCE", "label": "Inspection of warehouse and other facilities", "maxPoints": 5, "page": 37 }, + { "id": "EXP_SERVICE", "parentId": "EXPERIENCE", "label": "Customer Service", "maxPoints": 5, "page": 37 }, + { "id": "ORG_CAPACITY", "parentId": null, "label": "Proven Organizational Capacity", "maxPoints": 5, "page": 38 }, + { "id": "COMMUNITY", "parentId": null, "label": "Community Connection, Values Alignment", "maxPoints": 5, "page": 38 } + ], + + "criteriaNotes": { + "totalPoints": 100, + "nestingIsInconsistent": { + "$comment": [ + "This is the finding raised against #10. Under METHOD and EXPERIENCE the sub-items are", + "WEIGHTED sub-criteria summing exactly to the parent (15+10+5=30, 5+5+5+5=20).", + "Under ORG_CAPACITY and COMMUNITY the sub-bullets carry NO points and are guidance —", + "they are requirements mapped to the criterion, not sub-criteria. Identical visual", + "nesting, two different meanings, in one document." + ], + "weightedParents": ["METHOD", "EXPERIENCE"], + "guidanceOnlyParents": ["ORG_CAPACITY", "COMMUNITY"] + }, + "scoringLevelIsAmbiguous": { + "$comment": [ + "The document does not say whether a committee scores at parent or leaf level. Because", + "the binding mechanism is named-section (four sections), a response supplies ONE prose", + "block for METHOD — which argues for scoring at parent level. But the leaves carry", + "explicit weights, which argues for leaf level. Compare Tarrant County, which scores", + "flat top-level criteria only. #10 needs a rule." + ] + } + }, + + "requirements": [ + { "id": "R-A1", "kind": "mandatory_gate", "section": "A.1", "page": 4, "text": "Cover Page completed and signed.", "criterionIds": [] }, + { "id": "R-A2", "kind": "mandatory_gate", "section": "A.2", "page": 5, "text": "Bidder Checklist completed.", "criterionIds": [] }, + + { "id": "R-B2-LOCAL", "kind": "mandatory_gate", "section": "B.2", "page": 5, "text": "Certification of percentage of local foods to be provided.", "criterionIds": ["GEO_PREF"], "knownMissId": "KM-2" }, + { "id": "R-B6C", "kind": "mandatory_gate", "section": "B.6(c)", "page": 5, "text": "Two bid copies submitted.", "criterionIds": [] }, + { "id": "R-B6D", "kind": "mandatory_gate", "section": "B.6(d)", "page": 5, "text": "A copy of a current state or local health certificate assuring that all regulations for the preparation of food and the facilities are being met.", "criterionIds": ["EXP_WAREHOUSE"] }, + { "id": "R-B8-QUEST", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Distributor Pre-Qualification Questionnaire completed.", "criterionIds": ["EXP_KNOWLEDGE"] }, + { "id": "R-B8-LICENSE", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Business license.", "criterionIds": [] }, + { "id": "R-B8-OCCUPANCY", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Certificate of Occupancy.", "criterionIds": [] }, + { "id": "R-B8-FINANCIALS", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Two years of audited financial statements.", "criterionIds": ["ORG_CAPACITY"] }, + + { "id": "R-C1-VALUES", "kind": "mandatory_gate", "section": "C.1", "page": 5, "text": "Response to values statement.", "criterionIds": ["COMMUNITY"], "knownMissId": "KM-3" }, + { "id": "R-C5-INSPECTIONS", "kind": "mandatory_gate", "section": "C.5", "page": 5, "text": "Two most recent health inspections.", "criterionIds": ["EXP_WAREHOUSE"] }, + { "id": "R-D6-INSURANCE", "kind": "mandatory_gate", "section": "D.6", "page": 5, "text": "Insurance amounts.", "criterionIds": [] }, + + { "id": "R-SCHED-C", "kind": "pricing", "section": "Schedule C", "page": 41, "text": "Pricing Sheet — separate attachment to be completed in Excel.", "criterionIds": ["PRICING"], "knownMissId": "KM-1" }, + + { "id": "R-ATT-B", "kind": "certification_form", "section": "Attachment B", "page": 44, "text": "Certificate Regarding Debarment, Suspension, Ineligibility and Voluntary Exclusion.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-ATT-C", "kind": "certification_form", "section": "Attachment C", "page": 46, "text": "Certificate of Independent Price Determination.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-ATT-D", "kind": "certification_form", "section": "Attachment D", "page": 47, "text": "Permanent Certification Regarding Lobbying.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-ATT-A", "kind": "compliance_schedule", "section": "Attachment A", "page": 42, "text": "Healthy Schools Act requirements compliance.", "criterionIds": ["EXP_KNOWLEDGE"] }, + + { "id": "R-B7-FORMAT", "kind": "mandatory_gate", "section": "B.7", "page": 15, "text": "The submitted proposal must have four sections, titled: Executive Summary; Pricing; Method of Approach and Implementation Plan; Bidder's Experience, Expertise and Reliability.", "criterionIds": [] }, + + { "id": "R-C5-FOODSAFETY", "kind": "narrative", "section": "C.5", "page": 23, "text": "Warehouse and distribution food safety programme.", "criterionIds": ["EXP_WAREHOUSE", "FOOD_PACKAGING"] }, + { "id": "R-C6-PEST", "kind": "narrative", "section": "C.6", "page": 24, "text": "Stored products pest management programme.", "criterionIds": ["EXP_WAREHOUSE"] }, + { "id": "R-C7-QA", "kind": "narrative", "section": "C.7", "page": 24, "text": "Distributor's responsibility to provide quality assurance.", "criterionIds": ["FOOD_PACKAGING"] }, + { "id": "R-C8-FARM2SCHOOL", "kind": "narrative", "section": "C.8", "page": 25, "text": "Farm to School participation.", "criterionIds": ["GEO_PREF", "COMMUNITY"] }, + { "id": "R-C4-MENUS", "kind": "narrative", "section": "C.4", "page": 23, "text": "Use of advisory group and menus.", "criterionIds": ["EXP_SERVICE"] }, + { "id": "R-D1-DELIVERY", "kind": "narrative", "section": "D.1", "page": 26, "text": "Delivery requirements, including delivery windows per school site.", "criterionIds": ["EXP_DEPENDABILITY"] }, + { "id": "R-D3-RECORDS", "kind": "narrative", "section": "D.3", "page": 27, "text": "Record-keeping obligations.", "criterionIds": ["EXP_KNOWLEDGE"] }, + { "id": "R-USDA-FOODS", "kind": "narrative", "section": "B.2/C.3", "page": 7, "text": "Utilization of USDA Foods in the product offering, including processing and diversion.", "criterionIds": ["USDA_FOODS"] }, + { "id": "R-CN-LABELS", "kind": "narrative", "section": "B.7", "page": 15, "text": "How the Distributor will make nutritional information such as CN labels available to the SFA.", "criterionIds": ["FOOD_PACKAGING", "EXP_KNOWLEDGE"] }, + + { "id": "R-D8-EPA", "kind": "narrative", "section": "D.8", "page": 30, "text": "Economic price adjustment and price renegotiation terms.", "criterionIds": ["PRICING"] }, + { "id": "R-B5-PRICING-TERMS", "kind": "narrative", "section": "B.5", "page": 12, "text": "Requirements regarding pricing, bidding and ordering. Prices from the awarded bidder shall remain in effect for the term of the contract.", "criterionIds": ["PRICING"] }, + { "id": "R-E1-EEO", "kind": "certification_form", "section": "E.1", "page": 32, "text": "Equal Opportunity provisions.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-E3-CLEANAIR", "kind": "certification_form", "section": "E.3", "page": 33, "text": "Clean Air and Water Certification.", "criterionIds": [], "externalFormSchema": true } + ], + + "knownMisses": [ + { + "id": "KM-1", + "requirementId": "R-SCHED-C", + "type": "content_outside_document", + "page": 41, + "quotedText": "SCHEDULE C: PRICING SHEET *Separate attachment to be completed in Excel.", + "whatIsMissed": "All pricing structure and every price line. The obligation is present; its content is not.", + "whyNaiveExtractionFails": "The section exists and parses cleanly. Nothing signals that the substance is elsewhere — the extractor sees a heading and one sentence and reports success.", + "stakes": "Pricing is 40 of 100 points. An extraction that reports 'complete' here has silently dropped the largest scored component of the solicitation.", + "recoverableByBetterPdfParsing": false + }, + { + "id": "KM-2", + "requirementId": "R-B2-LOCAL", + "type": "obligation_buried_in_scoring_prose", + "page": 37, + "quotedText": "Geographic preference – use locally grown or raised foods to maximum extent possible ( Bidder must submit certification of percentage of locally grown or raised foods to be supplied. SFA reserves the right to audit the actual percentage used once the bid is awarded) – 5 points", + "whatIsMissed": "A MANDATORY submission stated only inside a parenthetical within a scored criterion's description.", + "whyNaiveExtractionFails": "An extractor walking Section K is enumerating criteria and weights, not obligations. The word 'must' sits inside a parenthesis in a scoring bullet. It appears on the Bidder Checklist as 'B.2' with no cross-reference to Section K, so neither location alone reveals that this is a gate.", + "stakes": "Missing it means a vendor is not told they will fail a checklist item, and the retailer's gate evaluation is incomplete.", + "recoverableByBetterPdfParsing": false, + "exercisesSweepBehaviour": "asserted_obligation — the second pass should assert an obligation in this span that no requirement covers" + }, + { + "id": "KM-3", + "requirementId": "R-C1-VALUES", + "type": "miskinded_as_informational", + "page": 5, + "quotedText": "C.1: Response to values statement", + "whatIsMissed": "Kind, not existence. 'Response to values statement' reads as narrative or informational; it is on the Bidder Checklist and is therefore a mandatory gate.", + "whyNaiveExtractionFails": "The text carries no obligation verb — no 'shall', no 'must'. Its mandatory status derives entirely from appearing on a checklist eleven pages earlier.", + "stakes": "Under #15's routing, a miskind sends this to a model for judgement instead of to code for a pass/fail check. Under #16 it would be bulk-accepted as informational and never individually confirmed.", + "recoverableByBetterPdfParsing": false, + "exercisesSweepBehaviour": "the sweep's second job — running over spans covered ONLY by informational requirements" + } + ], + + "expectedExtractionDifficulty": { + "rubricHasNoCriteriaHeading": { + "$comment": "The rubric lives under 'SECTION K: Evaluation Factors' → 'K.2 Technical Rating Scale'. Searching for 'evaluation criteria' or 'scoring' returns nothing. Weights are bullet suffixes ('– 40 points'), not a table.", + "page": 37 + }, + "checklistIsTheGateSource": { + "$comment": "Mandatory status for 17 items is established ONLY by the A.2 Bidder Checklist on page 5, using unicode ballot boxes. Each item is a terse cross-reference ('B.6(d)') whose obligation text lives elsewhere. Resolving gates requires joining the checklist to its targets.", + "page": 5 + }, + "textLayerIsClean": { + "$comment": "No page required OCR. This document does not exercise the OCR ladder — use SLC Airport for that." + } + } +} diff --git a/corpus/validate-truth.py b/corpus/validate-truth.py new file mode 100644 index 0000000..947c711 --- /dev/null +++ b/corpus/validate-truth.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Validate a corpus truth file's internal consistency. + +Run: python3 corpus/validate-truth.py corpus/