christian/grant-outreach-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ingestion): Phase 1 data spine — Grants.gov, PND RSS, NHDOJ registry, ProPublica enrichment

Browse Source 
Core: icpBandForRevenue (100K-5M primary), serverUpsertOrgFromRegistry
(case-insensitive name/city/state key, new-registrant signal),
serverEnrichOrg (IRS fields + re-band, all-null marks attempted),
serverListOrgsNeedingEnrichment.

Worker: four source verticals, each a thin fail-loud client + pure
tested normalize layer + DBOS scheduled workflow:
- ingestGrants (nightly): Search2 paginated (cap 1000) -> fetchOpportunity
  details (cap 200, logged drops, 250ms politeness) -> batched upsert
- ingestPndRss (nightly): RSS via fast-xml-parser, heuristic funder/
  deadline extraction, link-keyed upsert
- ingestNhdojOrgs (monthly): pdfjs-dist positioned-text extraction, pure
  row reconstruction (multi-line names, inferred columns, fail-loud on
  layout change), registry upsert; no-op warn when PDF URL unset
- enrichOrgs (daily): ProPublica search -> conservative name/city match
  (null beats guess) -> latest-filing revenue/NTEE/FYE, per-org steps for
  checkpointed resume, >20% batch failure rethrows

94 worker + 16 core + 5 ai tests green; docs/features/ingestion.md added.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
2026-07-16 13:11:50 -04:00
parent 14200edb60
commit 1735ff6754
35 changed files with 3427 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
yarn.lock
View File

@@ -1370,6 +1370,129 @@ __metadata:
languageName: node
linkType: hard
"@napi-rs/canvas-android-arm64@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-android-arm64@npm:0.1.100"
conditions: os=android & cpu=arm64
languageName: node
linkType: hard
"@napi-rs/canvas-darwin-arm64@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-darwin-arm64@npm:0.1.100"
conditions: os=darwin & cpu=arm64
languageName: node
linkType: hard
"@napi-rs/canvas-darwin-x64@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-darwin-x64@npm:0.1.100"
conditions: os=darwin & cpu=x64
languageName: node
linkType: hard
"@napi-rs/canvas-linux-arm-gnueabihf@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-linux-arm-gnueabihf@npm:0.1.100"
conditions: os=linux & cpu=arm
languageName: node
linkType: hard
"@napi-rs/canvas-linux-arm64-gnu@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-linux-arm64-gnu@npm:0.1.100"
conditions: os=linux & cpu=arm64 & libc=glibc
languageName: node
linkType: hard
"@napi-rs/canvas-linux-arm64-musl@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-linux-arm64-musl@npm:0.1.100"
conditions: os=linux & cpu=arm64 & libc=musl
languageName: node
linkType: hard
"@napi-rs/canvas-linux-riscv64-gnu@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-linux-riscv64-gnu@npm:0.1.100"
conditions: os=linux & cpu=riscv64 & libc=glibc
languageName: node
linkType: hard
"@napi-rs/canvas-linux-x64-gnu@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-linux-x64-gnu@npm:0.1.100"
conditions: os=linux & cpu=x64 & libc=glibc
languageName: node
linkType: hard
"@napi-rs/canvas-linux-x64-musl@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-linux-x64-musl@npm:0.1.100"
conditions: os=linux & cpu=x64 & libc=musl
languageName: node
linkType: hard
"@napi-rs/canvas-win32-arm64-msvc@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-win32-arm64-msvc@npm:0.1.100"
conditions: os=win32 & cpu=arm64
languageName: node
linkType: hard
"@napi-rs/canvas-win32-x64-msvc@npm:0.1.100":
version: 0.1.100
resolution: "@napi-rs/canvas-win32-x64-msvc@npm:0.1.100"
conditions: os=win32 & cpu=x64
languageName: node
linkType: hard
"@napi-rs/canvas@npm:^0.1.65":
version: 0.1.100
resolution: "@napi-rs/canvas@npm:0.1.100"
dependencies:
"@napi-rs/canvas-android-arm64": "npm:0.1.100"
"@napi-rs/canvas-darwin-arm64": "npm:0.1.100"
"@napi-rs/canvas-darwin-x64": "npm:0.1.100"
"@napi-rs/canvas-linux-arm-gnueabihf": "npm:0.1.100"
"@napi-rs/canvas-linux-arm64-gnu": "npm:0.1.100"
"@napi-rs/canvas-linux-arm64-musl": "npm:0.1.100"
"@napi-rs/canvas-linux-riscv64-gnu": "npm:0.1.100"
"@napi-rs/canvas-linux-x64-gnu": "npm:0.1.100"
"@napi-rs/canvas-linux-x64-musl": "npm:0.1.100"
"@napi-rs/canvas-win32-arm64-msvc": "npm:0.1.100"
"@napi-rs/canvas-win32-x64-msvc": "npm:0.1.100"
dependenciesMeta:
"@napi-rs/canvas-android-arm64":
optional: true
"@napi-rs/canvas-darwin-arm64":
optional: true
"@napi-rs/canvas-darwin-x64":
optional: true
"@napi-rs/canvas-linux-arm-gnueabihf":
optional: true
"@napi-rs/canvas-linux-arm64-gnu":
optional: true
"@napi-rs/canvas-linux-arm64-musl":
optional: true
"@napi-rs/canvas-linux-riscv64-gnu":
optional: true
"@napi-rs/canvas-linux-x64-gnu":
optional: true
"@napi-rs/canvas-linux-x64-musl":
optional: true
"@napi-rs/canvas-win32-arm64-msvc":
optional: true
"@napi-rs/canvas-win32-x64-msvc":
optional: true
canvas:
built: true
skia-canvas:
built: true
checksum: 10c0/d3dfca5620a41c34addd344bd4c448a5334d3f32f2562ec8390507b3897747ed1de9f08fcb9169cff6dd27111afb5b3eaf7e42cc1494c8fa4128e7d4235f6bea
languageName: node
linkType: hard
"@nodelib/fs.scandir@npm:2.1.5":
version: 2.1.5
resolution: "@nodelib/fs.scandir@npm:2.1.5"
@@ -1487,9 +1610,12 @@ __metadata:
"@types/pg": "npm:8.20.0"
drizzle-orm: "npm:0.44.6"
esbuild: "npm:^0.24.0"
fast-xml-parser: "npm:^4.5.0"
pdfjs-dist: "npm:^4.10.38"
pg: "npm:8.20.0"
tsx: "npm:^4.19.2"
typescript: "npm:^5.9.3"
vitest: "npm:^3.2.4"
languageName: unknown
linkType: soft
@@ -3580,6 +3706,17 @@ __metadata:
languageName: node
linkType: hard
"fast-xml-parser@npm:^4.5.0":
version: 4.5.7
resolution: "fast-xml-parser@npm:4.5.7"
dependencies:
strnum: "npm:^1.0.5"
bin:
fxparser: src/cli/cli.js
checksum: 10c0/5fccf3f53d6b2b83143d73089f04ef5db5343422891193cf10d92d3eb856007b7337818494109e46f6fa47b31b9716be15c4b275ff87a0aeb6f7315aa2edc181
languageName: node
linkType: hard
"fastq@npm:^1.6.0":
version: 1.20.1
resolution: "fastq@npm:1.20.1"
@@ -4899,6 +5036,18 @@ __metadata:
languageName: node
linkType: hard
"pdfjs-dist@npm:^4.10.38":
version: 4.10.38
resolution: "pdfjs-dist@npm:4.10.38"
dependencies:
"@napi-rs/canvas": "npm:^0.1.65"
dependenciesMeta:
"@napi-rs/canvas":
optional: true
checksum: 10c0/77b022109be7aac00372750a53decea3979409e6ef1cf93bf554351569cd4d1fafc70afae4a9a3e4b4de3facf59d3acd54d324b0fcff781374bcb00493d449ce
languageName: node
linkType: hard
"pg-cloudflare@npm:^1.3.0":
version: 1.4.0
resolution: "pg-cloudflare@npm:1.4.0"
@@ -5948,6 +6097,13 @@ __metadata:
languageName: node
linkType: hard
"strnum@npm:^1.0.5":
version: 1.1.2
resolution: "strnum@npm:1.1.2"
checksum: 10c0/a0fce2498fa3c64ce64a40dada41beb91cabe3caefa910e467dc0518ef2ebd7e4d10f8c2202a6104f1410254cae245066c0e94e2521fb4061a5cb41831952392
languageName: node
linkType: hard
"sucrase@npm:^3.35.0":
version: 3.35.1
resolution: "sucrase@npm:3.35.1"