Add Friendship PCS RFP ground truth and a truth-file validator

31 requirements (13 mandatory gates, 5 certification forms, 11 narrative,
1 pricing, 1 compliance schedule), 12 criteria across two levels, and three
labelled known-misses.

The known-misses are drawn from the document rather than invented:
  KM-1  Schedule C holds no prices; content is in an external Excel file,
        and pricing is 40 of 100 points
  KM-2  a mandatory submission stated only inside a parenthetical within a
        scored criterion's description in Section K
  KM-3  'Response to values statement' carries no obligation verb and is
        mandatory only by virtue of the checklist eleven pages earlier

validate-truth.py checks reference integrity and weight sums; these files
are hand-authored, so nothing else checks them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

corpus/friendship-pcs/rfp.truth.json

@@ -0,0 +1,182 @@
+{
+  "$comment": [
+    "Ground truth for corpus/friendship-pcs/rfp.pdf — what a correct extraction SHOULD produce.",
+    "Hand-authored from the source document, not generated. Where this disagrees with an",
+    "extractor, the extractor is wrong. See NOTES.md for how it was derived.",
+    "",
+    "Requirement ids are stable and are what response truth files answer against."
+  ],
+
+  "solicitation": {
+    "title": "Request for Proposal for the Supply and Delivery of Food Items",
+    "issuer": "Friendship Public Charter School",
+    "issuerLocation": "Washington, DC",
+    "issueDate": "2021-03-15",
+    "closingDate": "2021-04-16",
+    "pages": 52,
+    "programContext": ["NSLP", "SBP", "ASSP", "FFVP", "CACFP", "SFSP"],
+    "siteCount": 8
+  },
+
+  "bindingMechanism": {
+    "level": "named_section",
+    "$comment": [
+      "#22's coarse tier. B.7 mandates exactly four titled sections, which map to rubric",
+      "CATEGORIES but not to their sub-criteria. The 15/10/5 split inside Method of Approach",
+      "has no structural anchor in the response — all three are judged from one prose block.",
+      "This is the single most important fact about scoring this document."
+    ],
+    "requiredSections": [
+      { "ordinal": 1, "title": "Executive Summary", "mapsToCriterion": null },
+      { "ordinal": 2, "title": "Pricing", "mapsToCriterion": "PRICING" },
+      { "ordinal": 3, "title": "Method of Approach and Implementation Plan", "mapsToCriterion": "METHOD" },
+      { "ordinal": 4, "title": "Bidder's Experience, Expertise and Reliability", "mapsToCriterion": "EXPERIENCE" }
+    ]
+  },
+
+  "ratingScale": {
+    "$comment": "Per-document, never normalised (#10, #22). The prose anchor is load-bearing, not the integer.",
+    "min": 0,
+    "max": 5,
+    "anchors": [
+      { "points": 0, "label": "Completely Deficient", "anchor": "Elements is missing or is completely nonresponsive to the request" },
+      { "points": 1, "label": "Unsatisfactory", "anchor": "Fails to meet minimum requirements; major deficiencies which are not correctable" },
+      { "points": 2, "label": "Below Average", "anchor": "Marginally meets minimum requirements; significant deficiencies which may be correctable" },
+      { "points": 3, "label": "Average", "anchor": "Meets requirements; only minor deficiencies which are correctable" },
+      { "points": 4, "label": "Very Good", "anchor": "Meets requirements; no deficiencies" },
+      { "points": 5, "label": "Exceptional", "anchor": "Exceeds most, if not all requirements; no deficiencies" }
+    ]
+  },
+
+  "criteria": [
+    { "id": "PRICING", "parentId": null, "label": "Pricing", "maxPoints": 40, "page": 37 },
+    { "id": "METHOD", "parentId": null, "label": "Method of Approach and Implementation", "maxPoints": 30, "page": 37 },
+    { "id": "FOOD_PACKAGING", "parentId": "METHOD", "label": "Food and Packaging (Appearance, Quality, Taste and Acceptance)", "maxPoints": 15, "page": 37 },
+    { "id": "USDA_FOODS", "parentId": "METHOD", "label": "Utilization of USDA Foods", "maxPoints": 10, "page": 37 },
+    { "id": "GEO_PREF", "parentId": "METHOD", "label": "Geographic preference — locally grown or raised foods", "maxPoints": 5, "page": 37 },
+    { "id": "EXPERIENCE", "parentId": null, "label": "Bidder's Experience, Expertise, & Reliability", "maxPoints": 20, "page": 37 },
+    { "id": "EXP_KNOWLEDGE", "parentId": "EXPERIENCE", "label": "Knowledge of USDA, District requirements", "maxPoints": 5, "page": 37 },
+    { "id": "EXP_DEPENDABILITY", "parentId": "EXPERIENCE", "label": "Dependability (on-time deliveries, accuracy filling orders)", "maxPoints": 5, "page": 37 },
+    { "id": "EXP_WAREHOUSE", "parentId": "EXPERIENCE", "label": "Inspection of warehouse and other facilities", "maxPoints": 5, "page": 37 },
+    { "id": "EXP_SERVICE", "parentId": "EXPERIENCE", "label": "Customer Service", "maxPoints": 5, "page": 37 },
+    { "id": "ORG_CAPACITY", "parentId": null, "label": "Proven Organizational Capacity", "maxPoints": 5, "page": 38 },
+    { "id": "COMMUNITY", "parentId": null, "label": "Community Connection, Values Alignment", "maxPoints": 5, "page": 38 }
+  ],
+
+  "criteriaNotes": {
+    "totalPoints": 100,
+    "nestingIsInconsistent": {
+      "$comment": [
+        "This is the finding raised against #10. Under METHOD and EXPERIENCE the sub-items are",
+        "WEIGHTED sub-criteria summing exactly to the parent (15+10+5=30, 5+5+5+5=20).",
+        "Under ORG_CAPACITY and COMMUNITY the sub-bullets carry NO points and are guidance —",
+        "they are requirements mapped to the criterion, not sub-criteria. Identical visual",
+        "nesting, two different meanings, in one document."
+      ],
+      "weightedParents": ["METHOD", "EXPERIENCE"],
+      "guidanceOnlyParents": ["ORG_CAPACITY", "COMMUNITY"]
+    },
+    "scoringLevelIsAmbiguous": {
+      "$comment": [
+        "The document does not say whether a committee scores at parent or leaf level. Because",
+        "the binding mechanism is named-section (four sections), a response supplies ONE prose",
+        "block for METHOD — which argues for scoring at parent level. But the leaves carry",
+        "explicit weights, which argues for leaf level. Compare Tarrant County, which scores",
+        "flat top-level criteria only. #10 needs a rule."
+      ]
+    }
+  },
+
+  "requirements": [
+    { "id": "R-A1", "kind": "mandatory_gate", "section": "A.1", "page": 4, "text": "Cover Page completed and signed.", "criterionIds": [] },
+    { "id": "R-A2", "kind": "mandatory_gate", "section": "A.2", "page": 5, "text": "Bidder Checklist completed.", "criterionIds": [] },
+
+    { "id": "R-B2-LOCAL", "kind": "mandatory_gate", "section": "B.2", "page": 5, "text": "Certification of percentage of local foods to be provided.", "criterionIds": ["GEO_PREF"], "knownMissId": "KM-2" },
+    { "id": "R-B6C", "kind": "mandatory_gate", "section": "B.6(c)", "page": 5, "text": "Two bid copies submitted.", "criterionIds": [] },
+    { "id": "R-B6D", "kind": "mandatory_gate", "section": "B.6(d)", "page": 5, "text": "A copy of a current state or local health certificate assuring that all regulations for the preparation of food and the facilities are being met.", "criterionIds": ["EXP_WAREHOUSE"] },
+    { "id": "R-B8-QUEST", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Distributor Pre-Qualification Questionnaire completed.", "criterionIds": ["EXP_KNOWLEDGE"] },
+    { "id": "R-B8-LICENSE", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Business license.", "criterionIds": [] },
+    { "id": "R-B8-OCCUPANCY", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Certificate of Occupancy.", "criterionIds": [] },
+    { "id": "R-B8-FINANCIALS", "kind": "mandatory_gate", "section": "B.8", "page": 5, "text": "Two years of audited financial statements.", "criterionIds": ["ORG_CAPACITY"] },
+
+    { "id": "R-C1-VALUES", "kind": "mandatory_gate", "section": "C.1", "page": 5, "text": "Response to values statement.", "criterionIds": ["COMMUNITY"], "knownMissId": "KM-3" },
+    { "id": "R-C5-INSPECTIONS", "kind": "mandatory_gate", "section": "C.5", "page": 5, "text": "Two most recent health inspections.", "criterionIds": ["EXP_WAREHOUSE"] },
+    { "id": "R-D6-INSURANCE", "kind": "mandatory_gate", "section": "D.6", "page": 5, "text": "Insurance amounts.", "criterionIds": [] },
+
+    { "id": "R-SCHED-C", "kind": "pricing", "section": "Schedule C", "page": 41, "text": "Pricing Sheet — separate attachment to be completed in Excel.", "criterionIds": ["PRICING"], "knownMissId": "KM-1" },
+
+    { "id": "R-ATT-B", "kind": "certification_form", "section": "Attachment B", "page": 44, "text": "Certificate Regarding Debarment, Suspension, Ineligibility and Voluntary Exclusion.", "criterionIds": [], "externalFormSchema": true },
+    { "id": "R-ATT-C", "kind": "certification_form", "section": "Attachment C", "page": 46, "text": "Certificate of Independent Price Determination.", "criterionIds": [], "externalFormSchema": true },
+    { "id": "R-ATT-D", "kind": "certification_form", "section": "Attachment D", "page": 47, "text": "Permanent Certification Regarding Lobbying.", "criterionIds": [], "externalFormSchema": true },
+    { "id": "R-ATT-A", "kind": "compliance_schedule", "section": "Attachment A", "page": 42, "text": "Healthy Schools Act requirements compliance.", "criterionIds": ["EXP_KNOWLEDGE"] },
+
+    { "id": "R-B7-FORMAT", "kind": "mandatory_gate", "section": "B.7", "page": 15, "text": "The submitted proposal must have four sections, titled: Executive Summary; Pricing; Method of Approach and Implementation Plan; Bidder's Experience, Expertise and Reliability.", "criterionIds": [] },
+
+    { "id": "R-C5-FOODSAFETY", "kind": "narrative", "section": "C.5", "page": 23, "text": "Warehouse and distribution food safety programme.", "criterionIds": ["EXP_WAREHOUSE", "FOOD_PACKAGING"] },
+    { "id": "R-C6-PEST", "kind": "narrative", "section": "C.6", "page": 24, "text": "Stored products pest management programme.", "criterionIds": ["EXP_WAREHOUSE"] },
+    { "id": "R-C7-QA", "kind": "narrative", "section": "C.7", "page": 24, "text": "Distributor's responsibility to provide quality assurance.", "criterionIds": ["FOOD_PACKAGING"] },
+    { "id": "R-C8-FARM2SCHOOL", "kind": "narrative", "section": "C.8", "page": 25, "text": "Farm to School participation.", "criterionIds": ["GEO_PREF", "COMMUNITY"] },
+    { "id": "R-C4-MENUS", "kind": "narrative", "section": "C.4", "page": 23, "text": "Use of advisory group and menus.", "criterionIds": ["EXP_SERVICE"] },
+    { "id": "R-D1-DELIVERY", "kind": "narrative", "section": "D.1", "page": 26, "text": "Delivery requirements, including delivery windows per school site.", "criterionIds": ["EXP_DEPENDABILITY"] },
+    { "id": "R-D3-RECORDS", "kind": "narrative", "section": "D.3", "page": 27, "text": "Record-keeping obligations.", "criterionIds": ["EXP_KNOWLEDGE"] },
+    { "id": "R-USDA-FOODS", "kind": "narrative", "section": "B.2/C.3", "page": 7, "text": "Utilization of USDA Foods in the product offering, including processing and diversion.", "criterionIds": ["USDA_FOODS"] },
+    { "id": "R-CN-LABELS", "kind": "narrative", "section": "B.7", "page": 15, "text": "How the Distributor will make nutritional information such as CN labels available to the SFA.", "criterionIds": ["FOOD_PACKAGING", "EXP_KNOWLEDGE"] },
+
+    { "id": "R-D8-EPA", "kind": "narrative", "section": "D.8", "page": 30, "text": "Economic price adjustment and price renegotiation terms.", "criterionIds": ["PRICING"] },
+    { "id": "R-B5-PRICING-TERMS", "kind": "narrative", "section": "B.5", "page": 12, "text": "Requirements regarding pricing, bidding and ordering. Prices from the awarded bidder shall remain in effect for the term of the contract.", "criterionIds": ["PRICING"] },
+    { "id": "R-E1-EEO", "kind": "certification_form", "section": "E.1", "page": 32, "text": "Equal Opportunity provisions.", "criterionIds": [], "externalFormSchema": true },
+    { "id": "R-E3-CLEANAIR", "kind": "certification_form", "section": "E.3", "page": 33, "text": "Clean Air and Water Certification.", "criterionIds": [], "externalFormSchema": true }
+  ],
+
+  "knownMisses": [
+    {
+      "id": "KM-1",
+      "requirementId": "R-SCHED-C",
+      "type": "content_outside_document",
+      "page": 41,
+      "quotedText": "SCHEDULE C: PRICING SHEET  *Separate attachment to be completed in Excel.",
+      "whatIsMissed": "All pricing structure and every price line. The obligation is present; its content is not.",
+      "whyNaiveExtractionFails": "The section exists and parses cleanly. Nothing signals that the substance is elsewhere — the extractor sees a heading and one sentence and reports success.",
+      "stakes": "Pricing is 40 of 100 points. An extraction that reports 'complete' here has silently dropped the largest scored component of the solicitation.",
+      "recoverableByBetterPdfParsing": false
+    },
+    {
+      "id": "KM-2",
+      "requirementId": "R-B2-LOCAL",
+      "type": "obligation_buried_in_scoring_prose",
+      "page": 37,
+      "quotedText": "Geographic preference – use locally grown or raised foods to maximum extent possible ( Bidder must submit certification of percentage of locally grown or raised foods to be supplied. SFA reserves the right to audit the actual percentage used once the bid is awarded) – 5 points",
+      "whatIsMissed": "A MANDATORY submission stated only inside a parenthetical within a scored criterion's description.",
+      "whyNaiveExtractionFails": "An extractor walking Section K is enumerating criteria and weights, not obligations. The word 'must' sits inside a parenthesis in a scoring bullet. It appears on the Bidder Checklist as 'B.2' with no cross-reference to Section K, so neither location alone reveals that this is a gate.",
+      "stakes": "Missing it means a vendor is not told they will fail a checklist item, and the retailer's gate evaluation is incomplete.",
+      "recoverableByBetterPdfParsing": false,
+      "exercisesSweepBehaviour": "asserted_obligation — the second pass should assert an obligation in this span that no requirement covers"
+    },
+    {
+      "id": "KM-3",
+      "requirementId": "R-C1-VALUES",
+      "type": "miskinded_as_informational",
+      "page": 5,
+      "quotedText": "C.1: Response to values statement",
+      "whatIsMissed": "Kind, not existence. 'Response to values statement' reads as narrative or informational; it is on the Bidder Checklist and is therefore a mandatory gate.",
+      "whyNaiveExtractionFails": "The text carries no obligation verb — no 'shall', no 'must'. Its mandatory status derives entirely from appearing on a checklist eleven pages earlier.",
+      "stakes": "Under #15's routing, a miskind sends this to a model for judgement instead of to code for a pass/fail check. Under #16 it would be bulk-accepted as informational and never individually confirmed.",
+      "recoverableByBetterPdfParsing": false,
+      "exercisesSweepBehaviour": "the sweep's second job — running over spans covered ONLY by informational requirements"
+    }
+  ],
+
+  "expectedExtractionDifficulty": {
+    "rubricHasNoCriteriaHeading": {
+      "$comment": "The rubric lives under 'SECTION K: Evaluation Factors' → 'K.2 Technical Rating Scale'. Searching for 'evaluation criteria' or 'scoring' returns nothing. Weights are bullet suffixes ('– 40 points'), not a table.",
+      "page": 37
+    },
+    "checklistIsTheGateSource": {
+      "$comment": "Mandatory status for 17 items is established ONLY by the A.2 Bidder Checklist on page 5, using unicode ballot boxes. Each item is a terse cross-reference ('B.6(d)') whose obligation text lives elsewhere. Resolving gates requires joining the checklist to its targets.",
+      "page": 5
+    },
+    "textLayerIsClean": {
+      "$comment": "No page required OCR. This document does not exercise the OCR ladder — use SLC Airport for that."
+    }
+  }
+}

corpus/validate-truth.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Validate a corpus truth file's internal consistency.
+
+Run: python3 corpus/validate-truth.py corpus/<dir>/rfp.truth.json
+These files are hand-authored ground truth, so nothing else checks them.
+"""
+import json, sys
+
+path = sys.argv[1] if len(sys.argv) > 1 else 'friendship-pcs/rfp.truth.json'
+t = json.load(open(path))
+errs, warns = [], []
+
+crit = {c['id']: c for c in t['criteria']}
+reqs = {r['id']: r for r in t['requirements']}
+kms  = {k['id']: k for k in t['knownMisses']}
+nest = t['criteriaNotes']['nestingIsInconsistent']
+
+for r in t['requirements']:
+    for cid in r.get('criterionIds', []):
+        if cid not in crit: errs.append(f"{r['id']} -> unknown criterion {cid}")
+    if 'knownMissId' in r and r['knownMissId'] not in kms:
+        errs.append(f"{r['id']} -> unknown knownMiss {r['knownMissId']}")
+
+for k in t['knownMisses']:
+    if k['requirementId'] not in reqs:
+        errs.append(f"{k['id']} -> unknown requirement {k['requirementId']}")
+
+for c in t['criteria']:
+    if c['parentId'] and c['parentId'] not in crit:
+        errs.append(f"{c['id']} -> unknown parent {c['parentId']}")
+
+for pid in nest['weightedParents']:
+    kids = [c for c in t['criteria'] if c['parentId'] == pid]
+    s = sum(c['maxPoints'] for c in kids)
+    if s != crit[pid]['maxPoints']:
+        errs.append(f"{pid}: children sum {s} != parent {crit[pid]['maxPoints']}")
+    else:
+        print(f"  ok   {pid:14} weighted children sum {s} == parent")
+
+for pid in nest['guidanceOnlyParents']:
+    kids = [c for c in t['criteria'] if c['parentId'] == pid]
+    if kids: errs.append(f"{pid} declared guidance-only but has {len(kids)} child criteria")
+    else: print(f"  ok   {pid:14} guidance-only, no child criteria")
+
+top = sum(c['maxPoints'] for c in t['criteria'] if c['parentId'] is None)
+tot = t['criteriaNotes']['totalPoints']
+print(f"  {'ok  ' if top == tot else 'ERR '} top-level points sum {top} vs declared {tot}")
+if top != tot: errs.append(f"top-level sum {top} != {tot}")
+
+pts = sorted(a['points'] for a in t['ratingScale']['anchors'])
+if pts != list(range(t['ratingScale']['min'], t['ratingScale']['max'] + 1)):
+    errs.append(f"rating scale not contiguous: {pts}")
+else:
+    print(f"  ok   rating scale {pts[0]}-{pts[-1]}, {len(pts)} anchors")
+
+mapped = {c for r in t['requirements'] for c in r.get('criterionIds', [])}
+for cid, c in crit.items():
+    kids = [x for x in t['criteria'] if x['parentId'] == cid]
+    if cid not in mapped and not kids:
+        warns.append(f"criterion {cid} ({c['label'][:38]}) has no requirement mapped — unscoreable")
+
+sec = {s['mapsToCriterion'] for s in t['bindingMechanism']['requiredSections']} - {None}
+for cid in sec:
+    if cid not in crit: errs.append(f"bindingMechanism section -> unknown criterion {cid}")
+
+gates = [r for r in t['requirements'] if r['kind'] == 'mandatory_gate']
+kinds = {}
+for r in t['requirements']: kinds[r['kind']] = kinds.get(r['kind'], 0) + 1
+print(f"\n  requirements {len(t['requirements'])}  criteria {len(t['criteria'])}  knownMisses {len(t['knownMisses'])}")
+print(f"  by kind: " + ", ".join(f"{k}={v}" for k, v in sorted(kinds.items())))
+for w in warns: print(f"  WARN {w}")
+for e in errs: print(f"  ERR  {e}")
+print("\nVALID" if not errs else "\nINVALID")
+sys.exit(1 if errs else 0)