diff --git a/corpus/friendship-pcs/rfp.truth.json b/corpus/friendship-pcs/rfp.truth.json new file mode 100644 index 0000000..7f8d3cf --- /dev/null +++ b/corpus/friendship-pcs/rfp.truth.json @@ -0,0 +1,182 @@ +{ + "$comment": [ + "Ground truth for corpus/friendship-pcs/rfp.pdf — what a correct extraction SHOULD produce.", + "Hand-authored from the source document, not generated. Where this disagrees with an", + "extractor, the extractor is wrong. See NOTES.md for how it was derived.", + "", + "Requirement ids are stable and are what response truth files answer against." + ], + + "solicitation": { + "title": "Request for Proposal for the Supply and Delivery of Food Items", + "issuer": "Friendship Public Charter School", + "issuerLocation": "Washington, DC", + "issueDate": "2021-03-15", + "closingDate": "2021-04-16", + "pages": 52, + "programContext": ["NSLP", "SBP", "ASSP", "FFVP", "CACFP", "SFSP"], + "siteCount": 8 + }, + + "bindingMechanism": { + "level": "named_section", + "$comment": [ + "#22's coarse tier. B.7 mandates exactly four titled sections, which map to rubric", + "CATEGORIES but not to their sub-criteria. The 15/10/5 split inside Method of Approach", + "has no structural anchor in the response — all three are judged from one prose block.", + "This is the single most important fact about scoring this document." + ], + "requiredSections": [ + { "ordinal": 1, "title": "Executive Summary", "mapsToCriterion": null }, + { "ordinal": 2, "title": "Pricing", "mapsToCriterion": "PRICING" }, + { "ordinal": 3, "title": "Method of Approach and Implementation Plan", "mapsToCriterion": "METHOD" }, + { "ordinal": 4, "title": "Bidder's Experience, Expertise and Reliability", "mapsToCriterion": "EXPERIENCE" } + ] + }, + + "ratingScale": { + "$comment": "Per-document, never normalised (#10, #22). The prose anchor is load-bearing, not the integer.", + "min": 0, + "max": 5, + "anchors": [ + { "points": 0, "label": "Completely Deficient", "anchor": "Elements is missing or is completely nonresponsive to the request" }, + { "points": 1, "label": "Unsatisfactory", "anchor": "Fails to meet minimum requirements; major deficiencies which are not correctable" }, + { "points": 2, "label": "Below Average", "anchor": "Marginally meets minimum requirements; significant deficiencies which may be correctable" }, + { "points": 3, "label": "Average", "anchor": "Meets requirements; only minor deficiencies which are correctable" }, + { "points": 4, "label": "Very Good", "anchor": "Meets requirements; no deficiencies" }, + { "points": 5, "label": "Exceptional", "anchor": "Exceeds most, if not all requirements; no deficiencies" } + ] + }, + + "criteria": [ + { "id": "PRICING", "parentId": null, "label": "Pricing", "maxPoints": 40, "page": 37 }, + { "id": "METHOD", "parentId": null, "label": "Method of Approach and Implementation", "maxPoints": 30, "page": 37 }, + { "id": "FOOD_PACKAGING", "parentId": "METHOD", "label": "Food and Packaging (Appearance, Quality, Taste and Acceptance)", "maxPoints": 15, "page": 37 }, + { "id": "USDA_FOODS", "parentId": "METHOD", "label": "Utilization of USDA Foods", "maxPoints": 10, "page": 37 }, + { "id": "GEO_PREF", "parentId": "METHOD", "label": "Geographic preference — locally grown or raised foods", "maxPoints": 5, "page": 37 }, + { "id": "EXPERIENCE", "parentId": null, "label": "Bidder's Experience, Expertise, & Reliability", "maxPoints": 20, "page": 37 }, + { "id": "EXP_KNOWLEDGE", "parentId": "EXPERIENCE", "label": "Knowledge of USDA, District requirements", "maxPoints": 5, "page": 37 }, + { "id": "EXP_DEPENDABILITY", "parentId": "EXPERIENCE", "label": "Dependability (on-time deliveries, accuracy filling orders)", "maxPoints": 5, "page": 37 }, + { "id": "EXP_WAREHOUSE", "parentId": "EXPERIENCE", "label": "Inspection of warehouse and other facilities", "maxPoints": 5, "page": 37 }, + { "id": "EXP_SERVICE", "parentId": "EXPERIENCE", "label": "Customer Service", "maxPoints": 5, "page": 37 }, + { "id": "ORG_CAPACITY", "parentId": null, "label": "Proven Organizational Capacity", "maxPoints": 5, "page": 38 }, + { "id": "COMMUNITY", "parentId": null, "label": "Community Connection, Values Alignment", "maxPoints": 5, "page": 38 } + ], + + "criteriaNotes": { + "totalPoints": 100, + "nestingIsInconsistent": { + "$comment": [ + "This is the finding raised against #10. Under METHOD and EXPERIENCE the sub-items are", + "WEIGHTED sub-criteria summing exactly to the parent (15+10+5=30, 5+5+5+5=20).", + "Under ORG_CAPACITY and COMMUNITY the sub-bullets carry NO points and are guidance —", + "they are requirements mapped to the criterion, not sub-criteria. Identical visual", + "nesting, two different meanings, in one document." + ], + "weightedParents": ["METHOD", "EXPERIENCE"], + "guidanceOnlyParents": ["ORG_CAPACITY", "COMMUNITY"] + }, + "scoringLevelIsAmbiguous": { + "$comment": [ + "The document does not say whether a committee scores at parent or leaf level. Because", + "the binding mechanism is named-section (four sections), a response supplies ONE prose", + "block for METHOD — which argues for scoring at parent level. But the leaves carry", + "explicit weights, which argues for leaf level. Compare Tarrant County, which scores", + "flat top-level criteria only. #10 needs a rule." + ] + } + }, + + "requirements": [ + { "id": "R-A1", "kind": "mandatory_gate", "section": "A.1", "page": 4, "text": "Cover Page completed and signed.", "criterionIds": [] }, + { "id": "R-A2", "kind": "mandatory_gate", "section": "A.2", "page": 5, "text": "Bidder Checklist completed.", "criterionIds": [] }, + + { "id": "R-B2-LOCAL", "kind": "mandatory_gate", "section": "B.2", "page": 5, "text": "Certification of percentage of local foods to be provided.", "criterionIds": ["GEO_PREF"], "knownMissId": "KM-2" }, + { "id": "R-B6C", "kind": "mandatory_gate", "section": "B.6(c)", "page": 5, "text": "Two bid copies submitted.", "criterionIds": [] }, + { "id": "R-B6D", "kind": "mandatory_gate", "section": "B.6(d)", "page": 5, "text": "A copy of a current state or local health certificate assuring that all regulations for the preparation of food and the facilities are being met.", "criterionIds": ["EXP_WAREHOUSE"] }, + { "id": "R-B8-QUEST", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Distributor Pre-Qualification Questionnaire completed.", "criterionIds": ["EXP_KNOWLEDGE"] }, + { "id": "R-B8-LICENSE", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Business license.", "criterionIds": [] }, + { "id": "R-B8-OCCUPANCY", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Certificate of Occupancy.", "criterionIds": [] }, + { "id": "R-B8-FINANCIALS", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Two years of audited financial statements.", "criterionIds": ["ORG_CAPACITY"] }, + + { "id": "R-C1-VALUES", "kind": "mandatory_gate", "section": "C.1", "page": 5, "text": "Response to values statement.", "criterionIds": ["COMMUNITY"], "knownMissId": "KM-3" }, + { "id": "R-C5-INSPECTIONS", "kind": "mandatory_gate", "section": "C.5", "page": 5, "text": "Two most recent health inspections.", "criterionIds": ["EXP_WAREHOUSE"] }, + { "id": "R-D6-INSURANCE", "kind": "mandatory_gate", "section": "D.6", "page": 5, "text": "Insurance amounts.", "criterionIds": [] }, + + { "id": "R-SCHED-C", "kind": "pricing", "section": "Schedule C", "page": 41, "text": "Pricing Sheet — separate attachment to be completed in Excel.", "criterionIds": ["PRICING"], "knownMissId": "KM-1" }, + + { "id": "R-ATT-B", "kind": "certification_form", "section": "Attachment B", "page": 44, "text": "Certificate Regarding Debarment, Suspension, Ineligibility and Voluntary Exclusion.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-ATT-C", "kind": "certification_form", "section": "Attachment C", "page": 46, "text": "Certificate of Independent Price Determination.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-ATT-D", "kind": "certification_form", "section": "Attachment D", "page": 47, "text": "Permanent Certification Regarding Lobbying.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-ATT-A", "kind": "compliance_schedule", "section": "Attachment A", "page": 42, "text": "Healthy Schools Act requirements compliance.", "criterionIds": ["EXP_KNOWLEDGE"] }, + + { "id": "R-B7-FORMAT", "kind": "mandatory_gate", "section": "B.7", "page": 15, "text": "The submitted proposal must have four sections, titled: Executive Summary; Pricing; Method of Approach and Implementation Plan; Bidder's Experience, Expertise and Reliability.", "criterionIds": [] }, + + { "id": "R-C5-FOODSAFETY", "kind": "narrative", "section": "C.5", "page": 23, "text": "Warehouse and distribution food safety programme.", "criterionIds": ["EXP_WAREHOUSE", "FOOD_PACKAGING"] }, + { "id": "R-C6-PEST", "kind": "narrative", "section": "C.6", "page": 24, "text": "Stored products pest management programme.", "criterionIds": ["EXP_WAREHOUSE"] }, + { "id": "R-C7-QA", "kind": "narrative", "section": "C.7", "page": 24, "text": "Distributor's responsibility to provide quality assurance.", "criterionIds": ["FOOD_PACKAGING"] }, + { "id": "R-C8-FARM2SCHOOL", "kind": "narrative", "section": "C.8", "page": 25, "text": "Farm to School participation.", "criterionIds": ["GEO_PREF", "COMMUNITY"] }, + { "id": "R-C4-MENUS", "kind": "narrative", "section": "C.4", "page": 23, "text": "Use of advisory group and menus.", "criterionIds": ["EXP_SERVICE"] }, + { "id": "R-D1-DELIVERY", "kind": "narrative", "section": "D.1", "page": 26, "text": "Delivery requirements, including delivery windows per school site.", "criterionIds": ["EXP_DEPENDABILITY"] }, + { "id": "R-D3-RECORDS", "kind": "narrative", "section": "D.3", "page": 27, "text": "Record-keeping obligations.", "criterionIds": ["EXP_KNOWLEDGE"] }, + { "id": "R-USDA-FOODS", "kind": "narrative", "section": "B.2/C.3", "page": 7, "text": "Utilization of USDA Foods in the product offering, including processing and diversion.", "criterionIds": ["USDA_FOODS"] }, + { "id": "R-CN-LABELS", "kind": "narrative", "section": "B.7", "page": 15, "text": "How the Distributor will make nutritional information such as CN labels available to the SFA.", "criterionIds": ["FOOD_PACKAGING", "EXP_KNOWLEDGE"] }, + + { "id": "R-D8-EPA", "kind": "narrative", "section": "D.8", "page": 30, "text": "Economic price adjustment and price renegotiation terms.", "criterionIds": ["PRICING"] }, + { "id": "R-B5-PRICING-TERMS", "kind": "narrative", "section": "B.5", "page": 12, "text": "Requirements regarding pricing, bidding and ordering. Prices from the awarded bidder shall remain in effect for the term of the contract.", "criterionIds": ["PRICING"] }, + { "id": "R-E1-EEO", "kind": "certification_form", "section": "E.1", "page": 32, "text": "Equal Opportunity provisions.", "criterionIds": [], "externalFormSchema": true }, + { "id": "R-E3-CLEANAIR", "kind": "certification_form", "section": "E.3", "page": 33, "text": "Clean Air and Water Certification.", "criterionIds": [], "externalFormSchema": true } + ], + + "knownMisses": [ + { + "id": "KM-1", + "requirementId": "R-SCHED-C", + "type": "content_outside_document", + "page": 41, + "quotedText": "SCHEDULE C: PRICING SHEET *Separate attachment to be completed in Excel.", + "whatIsMissed": "All pricing structure and every price line. The obligation is present; its content is not.", + "whyNaiveExtractionFails": "The section exists and parses cleanly. Nothing signals that the substance is elsewhere — the extractor sees a heading and one sentence and reports success.", + "stakes": "Pricing is 40 of 100 points. An extraction that reports 'complete' here has silently dropped the largest scored component of the solicitation.", + "recoverableByBetterPdfParsing": false + }, + { + "id": "KM-2", + "requirementId": "R-B2-LOCAL", + "type": "obligation_buried_in_scoring_prose", + "page": 37, + "quotedText": "Geographic preference – use locally grown or raised foods to maximum extent possible ( Bidder must submit certification of percentage of locally grown or raised foods to be supplied. SFA reserves the right to audit the actual percentage used once the bid is awarded) – 5 points", + "whatIsMissed": "A MANDATORY submission stated only inside a parenthetical within a scored criterion's description.", + "whyNaiveExtractionFails": "An extractor walking Section K is enumerating criteria and weights, not obligations. The word 'must' sits inside a parenthesis in a scoring bullet. It appears on the Bidder Checklist as 'B.2' with no cross-reference to Section K, so neither location alone reveals that this is a gate.", + "stakes": "Missing it means a vendor is not told they will fail a checklist item, and the retailer's gate evaluation is incomplete.", + "recoverableByBetterPdfParsing": false, + "exercisesSweepBehaviour": "asserted_obligation — the second pass should assert an obligation in this span that no requirement covers" + }, + { + "id": "KM-3", + "requirementId": "R-C1-VALUES", + "type": "miskinded_as_informational", + "page": 5, + "quotedText": "C.1: Response to values statement", + "whatIsMissed": "Kind, not existence. 'Response to values statement' reads as narrative or informational; it is on the Bidder Checklist and is therefore a mandatory gate.", + "whyNaiveExtractionFails": "The text carries no obligation verb — no 'shall', no 'must'. Its mandatory status derives entirely from appearing on a checklist eleven pages earlier.", + "stakes": "Under #15's routing, a miskind sends this to a model for judgement instead of to code for a pass/fail check. Under #16 it would be bulk-accepted as informational and never individually confirmed.", + "recoverableByBetterPdfParsing": false, + "exercisesSweepBehaviour": "the sweep's second job — running over spans covered ONLY by informational requirements" + } + ], + + "expectedExtractionDifficulty": { + "rubricHasNoCriteriaHeading": { + "$comment": "The rubric lives under 'SECTION K: Evaluation Factors' → 'K.2 Technical Rating Scale'. Searching for 'evaluation criteria' or 'scoring' returns nothing. Weights are bullet suffixes ('– 40 points'), not a table.", + "page": 37 + }, + "checklistIsTheGateSource": { + "$comment": "Mandatory status for 17 items is established ONLY by the A.2 Bidder Checklist on page 5, using unicode ballot boxes. Each item is a terse cross-reference ('B.6(d)') whose obligation text lives elsewhere. Resolving gates requires joining the checklist to its targets.", + "page": 5 + }, + "textLayerIsClean": { + "$comment": "No page required OCR. This document does not exercise the OCR ladder — use SLC Airport for that." + } + } +} diff --git a/corpus/validate-truth.py b/corpus/validate-truth.py new file mode 100644 index 0000000..947c711 --- /dev/null +++ b/corpus/validate-truth.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Validate a corpus truth file's internal consistency. + +Run: python3 corpus/validate-truth.py corpus//rfp.truth.json +These files are hand-authored ground truth, so nothing else checks them. +""" +import json, sys + +path = sys.argv[1] if len(sys.argv) > 1 else 'friendship-pcs/rfp.truth.json' +t = json.load(open(path)) +errs, warns = [], [] + +crit = {c['id']: c for c in t['criteria']} +reqs = {r['id']: r for r in t['requirements']} +kms = {k['id']: k for k in t['knownMisses']} +nest = t['criteriaNotes']['nestingIsInconsistent'] + +for r in t['requirements']: + for cid in r.get('criterionIds', []): + if cid not in crit: errs.append(f"{r['id']} -> unknown criterion {cid}") + if 'knownMissId' in r and r['knownMissId'] not in kms: + errs.append(f"{r['id']} -> unknown knownMiss {r['knownMissId']}") + +for k in t['knownMisses']: + if k['requirementId'] not in reqs: + errs.append(f"{k['id']} -> unknown requirement {k['requirementId']}") + +for c in t['criteria']: + if c['parentId'] and c['parentId'] not in crit: + errs.append(f"{c['id']} -> unknown parent {c['parentId']}") + +for pid in nest['weightedParents']: + kids = [c for c in t['criteria'] if c['parentId'] == pid] + s = sum(c['maxPoints'] for c in kids) + if s != crit[pid]['maxPoints']: + errs.append(f"{pid}: children sum {s} != parent {crit[pid]['maxPoints']}") + else: + print(f" ok {pid:14} weighted children sum {s} == parent") + +for pid in nest['guidanceOnlyParents']: + kids = [c for c in t['criteria'] if c['parentId'] == pid] + if kids: errs.append(f"{pid} declared guidance-only but has {len(kids)} child criteria") + else: print(f" ok {pid:14} guidance-only, no child criteria") + +top = sum(c['maxPoints'] for c in t['criteria'] if c['parentId'] is None) +tot = t['criteriaNotes']['totalPoints'] +print(f" {'ok ' if top == tot else 'ERR '} top-level points sum {top} vs declared {tot}") +if top != tot: errs.append(f"top-level sum {top} != {tot}") + +pts = sorted(a['points'] for a in t['ratingScale']['anchors']) +if pts != list(range(t['ratingScale']['min'], t['ratingScale']['max'] + 1)): + errs.append(f"rating scale not contiguous: {pts}") +else: + print(f" ok rating scale {pts[0]}-{pts[-1]}, {len(pts)} anchors") + +mapped = {c for r in t['requirements'] for c in r.get('criterionIds', [])} +for cid, c in crit.items(): + kids = [x for x in t['criteria'] if x['parentId'] == cid] + if cid not in mapped and not kids: + warns.append(f"criterion {cid} ({c['label'][:38]}) has no requirement mapped — unscoreable") + +sec = {s['mapsToCriterion'] for s in t['bindingMechanism']['requiredSections']} - {None} +for cid in sec: + if cid not in crit: errs.append(f"bindingMechanism section -> unknown criterion {cid}") + +gates = [r for r in t['requirements'] if r['kind'] == 'mandatory_gate'] +kinds = {} +for r in t['requirements']: kinds[r['kind']] = kinds.get(r['kind'], 0) + 1 +print(f"\n requirements {len(t['requirements'])} criteria {len(t['criteria'])} knownMisses {len(t['knownMisses'])}") +print(f" by kind: " + ", ".join(f"{k}={v}" for k, v in sorted(kinds.items()))) +for w in warns: print(f" WARN {w}") +for e in errs: print(f" ERR {e}") +print("\nVALID" if not errs else "\nINVALID") +sys.exit(1 if errs else 0)