diff --git a/apps/outreach-worker/package.json b/apps/outreach-worker/package.json index fff53de..9f22f53 100644 --- a/apps/outreach-worker/package.json +++ b/apps/outreach-worker/package.json @@ -21,6 +21,7 @@ "@novelpad/outreach-core": "workspace:^", "drizzle-orm": "0.44.6", "fast-xml-parser": "^4.5.0", + "fflate": "^0.8.2", "pdfjs-dist": "^4.10.38", "pg": "8.20.0", "tsx": "^4.19.2" diff --git a/apps/outreach-worker/src/main.ts b/apps/outreach-worker/src/main.ts index ba440e8..11ca38f 100644 --- a/apps/outreach-worker/src/main.ts +++ b/apps/outreach-worker/src/main.ts @@ -34,6 +34,7 @@ import { setEmbedGrantsDeps } from './workflows/embed-grants.js'; import { setEnrichOrgsDeps } from './workflows/enrich-orgs.js'; import { setExpireGrantsDeps } from './workflows/expire-grants.js'; import { setIngestGrantsDeps } from './workflows/ingest-grants.js'; +import { setIngest990pfDeps } from './workflows/ingest-990pf.js'; import { setIngestNhdojOrgsDeps } from './workflows/ingest-nhdoj-orgs.js'; import { setMatchGrantsDeps } from './workflows/match-grants.js'; import { setIngestPndRssDeps } from './workflows/ingest-pnd-rss.js'; @@ -69,6 +70,7 @@ async function main() { setEnrichOrgsDeps({ db }); setEmbedGrantsDeps({ db }); setMatchGrantsDeps({ db }); + setIngest990pfDeps({ db }); DBOS.setConfig({ name: 'helmdocs-outreach-worker', diff --git a/apps/outreach-worker/src/run-once.ts b/apps/outreach-worker/src/run-once.ts index fcad4c2..e346afc 100644 --- a/apps/outreach-worker/src/run-once.ts +++ b/apps/outreach-worker/src/run-once.ts @@ -32,6 +32,10 @@ import { runIngestGrantsNow, setIngestGrantsDeps, } from './workflows/ingest-grants.js'; +import { + runIngest990PfNow, + setIngest990pfDeps, +} from './workflows/ingest-990pf.js'; import { runIngestNhdojOrgsNow, setIngestNhdojOrgsDeps, @@ -53,6 +57,7 @@ const RUNNERS: Record Promise> = { expireGrants: runExpireGrantsNow, embedGrants: runEmbedGrantsNow, matchGrants: runMatchGrantsNow, + ingest990pf: runIngest990PfNow, }; const FIRST_RUN_ORDER = [ @@ -61,6 +66,7 @@ const FIRST_RUN_ORDER = [ 'ingestNhdojOrgs', 'expireGrants', 'enrichOrgs', + 'ingest990pf', 'embedGrants', 'matchGrants', ]; @@ -99,6 +105,7 @@ async function main() { setEnrichOrgsDeps({ db }); setEmbedGrantsDeps({ db }); setMatchGrantsDeps({ db }); + setIngest990pfDeps({ db }); DBOS.setConfig({ name: 'helmdocs-outreach-worker', diff --git a/apps/outreach-worker/src/sources/irs-990pf/bmf.test.ts b/apps/outreach-worker/src/sources/irs-990pf/bmf.test.ts new file mode 100644 index 0000000..eea0de3 --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/bmf.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest'; + +import { parseBmfFoundations, parseCsvLine } from './bmf.js'; + +const HEADER = + 'EIN,NAME,ICO,STREET,CITY,STATE,ZIP,GROUP,SUBSECTION,AFFILIATION,CLASSIFICATION,RULING,DEDUCTIBILITY,FOUNDATION,ACTIVITY,ORGANIZATION,STATUS,TAX_PERIOD,ASSET_CD,INCOME_CD,FILING_REQ_CD,PF_FILING_REQ_CD,ACCT_PD,ASSET_AMT,INCOME_AMT,REVENUE_AMT,NTEE_CD,SORT_NAME'; + +function row(fields: Partial>): string { + const cols = HEADER.split(','); + return cols.map((c) => fields[c] ?? '').join(','); +} + +describe('parseCsvLine', () => { + it('splits an unquoted line on commas', () => { + expect(parseCsvLine('a,b,c')).toEqual(['a', 'b', 'c']); + }); + + it('handles a quoted field containing a comma', () => { + expect(parseCsvLine('a,"b, still b",c')).toEqual(['a', 'b, still b', 'c']); + }); + + it('unescapes doubled quotes inside a quoted field', () => { + expect(parseCsvLine('a,"he said ""hi""",c')).toEqual([ + 'a', + 'he said "hi"', + 'c', + ]); + }); + + it('handles an empty trailing field', () => { + expect(parseCsvLine('a,b,')).toEqual(['a', 'b', '']); + }); +}); + +describe('parseBmfFoundations', () => { + it('keeps only rows with PF_FILING_REQ_CD === "1"', () => { + const csv = [ + HEADER, + row({ + EIN: '020123456', + NAME: 'Granite State Foundation', + CITY: 'Manchester', + STATE: 'NH', + NTEE_CD: 'T20', + PF_FILING_REQ_CD: '1', + ASSET_AMT: '5000000', + }), + row({ + EIN: '020654321', + NAME: 'Not A Private Foundation Inc', + CITY: 'Concord', + STATE: 'NH', + PF_FILING_REQ_CD: '0', + ASSET_AMT: '1000000', + }), + ].join('\n'); + + const foundations = parseBmfFoundations(csv); + expect(foundations).toHaveLength(1); + expect(foundations[0]?.name).toBe('Granite State Foundation'); + }); + + it('zero-pads a short EIN to 9 digits', () => { + const csv = [ + HEADER, + row({ + EIN: '123456', // 6 digits, as BMF sometimes drops leading zeros + NAME: 'Small EIN Foundation', + PF_FILING_REQ_CD: '1', + }), + ].join('\n'); + + const foundations = parseBmfFoundations(csv); + expect(foundations[0]?.ein).toBe('000123456'); + }); + + it('parses a quoted NAME field containing a comma', () => { + const csv = [ + HEADER, + row({ + EIN: '020123456', + NAME: '"Smith, Jones & Family Foundation"', + CITY: 'Nashua', + STATE: 'NH', + PF_FILING_REQ_CD: '1', + ASSET_AMT: '2500000', + NTEE_CD: 'T30', + }), + ].join('\n'); + + const foundations = parseBmfFoundations(csv); + expect(foundations[0]?.name).toBe('Smith, Jones & Family Foundation'); + expect(foundations[0]?.city).toBe('Nashua'); + expect(foundations[0]?.totalAssets).toBe(2_500_000); + }); + + it('maps blank optional fields to null', () => { + const csv = [ + HEADER, + row({ + EIN: '020999999', + NAME: 'No Extras Foundation', + PF_FILING_REQ_CD: '1', + }), + ].join('\n'); + + const foundations = parseBmfFoundations(csv); + expect(foundations[0]).toMatchObject({ + city: null, + state: null, + nteeCode: null, + totalAssets: null, + }); + }); + + it('skips rows with a blank EIN or blank NAME', () => { + const csv = [ + HEADER, + row({ EIN: '', NAME: 'No EIN Foundation', PF_FILING_REQ_CD: '1' }), + row({ EIN: '020111111', NAME: '', PF_FILING_REQ_CD: '1' }), + ].join('\n'); + + expect(parseBmfFoundations(csv)).toEqual([]); + }); + + it('returns an empty array for a header-only CSV', () => { + expect(parseBmfFoundations(HEADER)).toEqual([]); + }); + + it('throws when required columns are missing from the header', () => { + expect(() => parseBmfFoundations('FOO,BAR\n1,2')).toThrow( + /missing required column/i, + ); + }); +}); diff --git a/apps/outreach-worker/src/sources/irs-990pf/bmf.ts b/apps/outreach-worker/src/sources/irs-990pf/bmf.ts new file mode 100644 index 0000000..e68ca0b --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/bmf.ts @@ -0,0 +1,144 @@ +/** + * IRS Business Master File (BMF) state extract — the discovery layer for + * the 990-PF funder-precedent index. The BMF lists every EO-recognized + * organization in a state, one row per EIN, including whether it's a + * private foundation required to file a 990-PF (`PF_FILING_REQ_CD === '1'`). + * + * `fetchBmfCsv` is the only impure piece; `parseBmfFoundations` is a pure + * function of CSV text so it's unit-testable on fixture strings. + */ + +export interface BmfFoundation { + /** Zero-padded to 9 digits, e.g. "020123456". */ + readonly ein: string; + readonly name: string; + readonly city: string | null; + readonly state: string | null; + readonly nteeCode: string | null; + readonly totalAssets: number | null; +} + +/** + * Tolerant CSV line splitter: handles double-quoted fields (with embedded + * commas and escaped `""`) defensively, even though the IRS BMF export's + * fields are unquoted in practice. + */ +export function parseCsvLine(line: string): string[] { + const fields: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (inQuotes) { + if (char === '"') { + if (line[i + 1] === '"') { + current += '"'; + i++; + } else { + inQuotes = false; + } + } else { + current += char; + } + } else if (char === '"') { + inQuotes = true; + } else if (char === ',') { + fields.push(current); + current = ''; + } else { + current += char; + } + } + fields.push(current); + return fields; +} + +const REQUIRED_COLUMNS = ['EIN', 'NAME', 'PF_FILING_REQ_CD'] as const; + +/** + * Parses a BMF state CSV into the private-foundation subset — + * `PF_FILING_REQ_CD === '1'` — that this vertical cares about. Column + * lookup is by header name (not position), since the BMF export carries + * many more columns than we use. + * + * Malformed/short rows (fewer fields than the header, blank EIN or name) + * are skipped rather than thrown on — a single-row anomaly in a 1.6MB + * government CSV shouldn't fail the whole discovery pass. + */ +export function parseBmfFoundations(csv: string): BmfFoundation[] { + const lines = csv.split(/\r?\n/).filter((line) => line.trim().length > 0); + if (lines.length === 0) return []; + + const header = parseCsvLine(lines[0]!).map((h) => h.trim().toUpperCase()); + const einIdx = header.indexOf('EIN'); + const nameIdx = header.indexOf('NAME'); + const cityIdx = header.indexOf('CITY'); + const stateIdx = header.indexOf('STATE'); + const nteeIdx = header.indexOf('NTEE_CD'); + const pfReqIdx = header.indexOf('PF_FILING_REQ_CD'); + const assetIdx = header.indexOf('ASSET_AMT'); + + const missing = REQUIRED_COLUMNS.filter((col) => header.indexOf(col) === -1); + if (missing.length > 0) { + throw new Error( + `parseBmfFoundations: BMF CSV header is missing required column(s): ${missing.join(', ')}`, + ); + } + + const out: BmfFoundation[] = []; + for (let i = 1; i < lines.length; i++) { + const fields = parseCsvLine(lines[i]!); + + const pfCode = fields[pfReqIdx]?.trim(); + if (pfCode !== '1') continue; // not a 990-PF filer + + const rawEin = fields[einIdx]?.trim() ?? ''; + const digits = rawEin.replace(/\D/g, ''); + if (digits === '') continue; + const ein = digits.padStart(9, '0'); + + const name = fields[nameIdx]?.trim() ?? ''; + if (name === '') continue; + + const city = cityIdx === -1 ? null : fields[cityIdx]?.trim() || null; + const state = stateIdx === -1 ? null : fields[stateIdx]?.trim() || null; + const nteeCode = nteeIdx === -1 ? null : fields[nteeIdx]?.trim() || null; + + let totalAssets: number | null = null; + if (assetIdx !== -1) { + const rawAsset = fields[assetIdx]?.trim(); + if (rawAsset != null && rawAsset !== '') { + const parsed = Number(rawAsset); + totalAssets = Number.isFinite(parsed) ? Math.round(parsed) : null; + } + } + + out.push({ ein, name, city, state, nteeCode, totalAssets }); + } + + return out; +} + +/** + * Fetches a state's IRS BMF extract. irs.gov sits behind Akamai bot + * detection — bare/default clients get a 403; Node's fetch with + * browser-like headers passes (same approach as + * `ingest-nhdoj-orgs.ts`'s `fetchNhdojRegistryPdf`, verified 2026-07-16). + */ +export async function fetchBmfCsv(state = 'nh'): Promise { + const url = `https://www.irs.gov/pub/irs-soi/eo_${state}.csv`; + const res = await fetch(url, { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', + Accept: 'text/csv,*/*', + }, + }); + if (!res.ok) { + throw new Error( + `fetchBmfCsv: ${url} responded ${res.status} ${res.statusText}`, + ); + } + return res.text(); +} diff --git a/apps/outreach-worker/src/sources/irs-990pf/index-csv.test.ts b/apps/outreach-worker/src/sources/irs-990pf/index-csv.test.ts new file mode 100644 index 0000000..3384fdf --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/index-csv.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest'; + +import { selectFilings } from './index-csv.js'; + +const HEADER = + 'RETURN_ID,FILING_TYPE,EIN,TAX_PERIOD,SUB_DATE,TAXPAYER_NAME,RETURN_TYPE,DLN,OBJECT_ID,XML_BATCH_ID'; + +function row(fields: { + returnId?: string; + filingType?: string; + ein: string; + taxPeriod: string; + subDate?: string; + taxpayerName?: string; + returnType: string; + dln?: string; + objectId: string; + batchId: string; +}): string { + return [ + fields.returnId ?? '1', + fields.filingType ?? 'EFILE', + fields.ein, + fields.taxPeriod, + fields.subDate ?? '20270301', + fields.taxpayerName ?? 'SOME FOUNDATION', + fields.returnType, + fields.dln ?? '93000000000000', + fields.objectId, + fields.batchId, + ].join(','); +} + +describe('selectFilings', () => { + it('keeps only RETURN_TYPE === "990PF" rows', () => { + const csv = [ + HEADER, + row({ + ein: '020123456', + taxPeriod: '202612', + returnType: '990PF', + objectId: '111', + batchId: 'BATCH1', + }), + row({ + ein: '020654321', + taxPeriod: '202612', + returnType: '990', + objectId: '222', + batchId: 'BATCH1', + }), + ].join('\n'); + + const targetEins = new Set(['020123456', '020654321']); + const filings = selectFilings(csv, targetEins); + expect(filings).toHaveLength(1); + expect(filings[0]?.ein).toBe('020123456'); + }); + + it('filters to the target EIN set', () => { + const csv = [ + HEADER, + row({ + ein: '020999999', + taxPeriod: '202612', + returnType: '990PF', + objectId: '333', + batchId: 'BATCH1', + }), + ].join('\n'); + + expect(selectFilings(csv, new Set(['020123456']))).toEqual([]); + }); + + it('zero-pads a short EIN before matching against the target set', () => { + const csv = [ + HEADER, + row({ + ein: '20123456', // 8 digits + taxPeriod: '202612', + returnType: '990PF', + objectId: '444', + batchId: 'BATCH1', + }), + ].join('\n'); + + const filings = selectFilings(csv, new Set(['020123456'])); + expect(filings).toHaveLength(1); + expect(filings[0]?.ein).toBe('020123456'); + }); + + it('keeps only the LATEST TAX_PERIOD per EIN within the file', () => { + const csv = [ + HEADER, + row({ + ein: '020123456', + taxPeriod: '202512', + returnType: '990PF', + objectId: 'OLD', + batchId: 'BATCH1', + }), + row({ + ein: '020123456', + taxPeriod: '202612', + returnType: '990PF', + objectId: 'NEW', + batchId: 'BATCH2', + }), + ].join('\n'); + + const filings = selectFilings(csv, new Set(['020123456'])); + expect(filings).toHaveLength(1); + expect(filings[0]?.objectId).toBe('NEW'); + expect(filings[0]?.batchId).toBe('BATCH2'); + }); + + it('skips rows with a blank OBJECT_ID or XML_BATCH_ID', () => { + const csv = [ + HEADER, + row({ + ein: '020123456', + taxPeriod: '202612', + returnType: '990PF', + objectId: '', + batchId: 'BATCH1', + }), + ].join('\n'); + + expect(selectFilings(csv, new Set(['020123456']))).toEqual([]); + }); + + it('returns an empty array for a header-only file', () => { + expect(selectFilings(HEADER, new Set(['020123456']))).toEqual([]); + }); + + it('throws when required columns are missing from the header', () => { + expect(() => selectFilings('FOO,BAR\n1,2', new Set(['020123456']))).toThrow( + /missing required column/i, + ); + }); +}); diff --git a/apps/outreach-worker/src/sources/irs-990pf/index-csv.ts b/apps/outreach-worker/src/sources/irs-990pf/index-csv.ts new file mode 100644 index 0000000..4d8cb9d --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/index-csv.ts @@ -0,0 +1,143 @@ +/** + * IRS e-file index CSV — maps EINs to the batch ZIP + XML object id + * containing their 990-PF filing for a given year. + * (`https://apps.irs.gov/pub/epostcard/990/xml/{year}/index_{year}.csv`). + * + * These files run ~28MB, so `selectFilings` walks the text line-by-line + * (via a manual index scan, not `String#split` — no ~300K-element line + * array materialized) and only ever retains the handful of rows that match + * our target EIN set, never a parsed copy of the whole file. + */ +import { parseCsvLine } from '#~/sources/irs-990pf/bmf.js'; + +export interface SelectedFiling { + /** Zero-padded to 9 digits. */ + readonly ein: string; + readonly objectId: string; + readonly batchId: string; + /** Wire value from `TAX_PERIOD`, typically `YYYYMM`. */ + readonly taxPeriod: string; +} + +const REQUIRED_COLUMNS = [ + 'EIN', + 'TAX_PERIOD', + 'RETURN_TYPE', + 'OBJECT_ID', + 'XML_BATCH_ID', +] as const; + +/** Yields each line of `text` without ever allocating a full line array. */ +function* iterateLines(text: string): Generator { + const len = text.length; + let start = 0; + while (start < len) { + let end = text.indexOf('\n', start); + if (end === -1) end = len; + let line = text.slice(start, end); + if (line.endsWith('\r')) line = line.slice(0, -1); + if (line.length > 0) yield line; + start = end + 1; + } +} + +/** + * Selects `RETURN_TYPE === '990PF'` rows whose EIN is in `targetEins`, + * keeping only the LATEST `TAX_PERIOD` per EIN found in this file (an EIN + * can appear more than once in a year's index across amended/superseded + * filings — the largest `TAX_PERIOD` wins). + */ +export function selectFilings( + csvText: string, + targetEins: ReadonlySet, +): SelectedFiling[] { + const latestByEin = new Map< + string, + SelectedFiling & { taxPeriodNum: number } + >(); + + let header: string[] | null = null; + let einIdx = -1; + let taxPeriodIdx = -1; + let returnTypeIdx = -1; + let objectIdIdx = -1; + let batchIdIdx = -1; + + for (const line of iterateLines(csvText)) { + if (header == null) { + header = parseCsvLine(line).map((h) => h.trim().toUpperCase()); + einIdx = header.indexOf('EIN'); + taxPeriodIdx = header.indexOf('TAX_PERIOD'); + returnTypeIdx = header.indexOf('RETURN_TYPE'); + objectIdIdx = header.indexOf('OBJECT_ID'); + batchIdIdx = header.indexOf('XML_BATCH_ID'); + + const missing = REQUIRED_COLUMNS.filter( + (col) => header!.indexOf(col) === -1, + ); + if (missing.length > 0) { + throw new Error( + `selectFilings: e-file index CSV header is missing required column(s): ${missing.join(', ')}`, + ); + } + continue; + } + + const fields = parseCsvLine(line); + const returnType = fields[returnTypeIdx]?.trim(); + if (returnType !== '990PF') continue; + + const rawEin = fields[einIdx]?.trim() ?? ''; + const digits = rawEin.replace(/\D/g, ''); + if (digits === '') continue; + const ein = digits.padStart(9, '0'); + if (!targetEins.has(ein)) continue; + + const objectId = fields[objectIdIdx]?.trim() ?? ''; + const batchId = fields[batchIdIdx]?.trim() ?? ''; + if (objectId === '' || batchId === '') continue; + + const taxPeriod = fields[taxPeriodIdx]?.trim() ?? ''; + const taxPeriodNum = Number(taxPeriod); + + const existing = latestByEin.get(ein); + if ( + existing == null || + (Number.isFinite(taxPeriodNum) && taxPeriodNum > existing.taxPeriodNum) + ) { + latestByEin.set(ein, { + ein, + objectId, + batchId, + taxPeriod, + taxPeriodNum: Number.isFinite(taxPeriodNum) ? taxPeriodNum : -1, + }); + } + } + + return [...latestByEin.values()].map( + ({ taxPeriodNum: _taxPeriodNum, ...rest }) => rest, + ); +} + +/** + * Fetches one year's e-file index CSV. Same Akamai-friendly headers as + * `fetchBmfCsv`/`fetchNhdojRegistryPdf` — apps.irs.gov is also + * Akamai-fronted (verified 2026-07-16). + */ +export async function fetchIndexCsv(year: number): Promise { + const url = `https://apps.irs.gov/pub/epostcard/990/xml/${year}/index_${year}.csv`; + const res = await fetch(url, { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', + Accept: 'text/csv,*/*', + }, + }); + if (!res.ok) { + throw new Error( + `fetchIndexCsv: ${url} responded ${res.status} ${res.statusText}`, + ); + } + return res.text(); +} diff --git a/apps/outreach-worker/src/sources/irs-990pf/parse-990pf-xml.test.ts b/apps/outreach-worker/src/sources/irs-990pf/parse-990pf-xml.test.ts new file mode 100644 index 0000000..e870189 --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/parse-990pf-xml.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from 'vitest'; + +import { parse990PfXml } from './parse-990pf-xml.js'; + +// Realistic (trimmed) 990-PF e-file XML shape, namespace-prefixed the way +// the real IRS MeF documents are (`irs:` on IRS990PF-specific elements) — +// `removeNSPrefix: true` should strip these transparently. +const MULTI_GRANT_FIXTURE = ` + + + 2026 + 2026-12-31 + + + + + + See attached list + Submit a two-page letter of inquiry. + March 1 and September 1 annually. + Grants limited to 501(c)(3) organizations in New Hampshire. + + + + Granite State Youth Services + + + Manchester + NH + + 25000 + General operating support + + + Jane Q. Public + + Concord + NH + + 5000 + Scholarship + + + + +`; + +// A filer with exactly one grant paid — fast-xml-parser collapses a +// single-element repeated group to a bare object without `isArray`. +const SINGLE_GRANT_FIXTURE = ` + + + 2025 + + + + + + + Lone Grantee Org + + 1200 + + + + +`; + +const NO_GRANTS_FIXTURE = ` + + + 2024-06-30 + + + + + + +`; + +describe('parse990PfXml', () => { + it('extracts grants paid, including a business-name recipient', () => { + const parsed = parse990PfXml(MULTI_GRANT_FIXTURE); + expect(parsed.grantsPaid).toHaveLength(2); + expect(parsed.grantsPaid[0]).toEqual({ + recipientName: 'Granite State Youth Services', + recipientCity: 'Manchester', + recipientState: 'NH', + amount: 25000, + purpose: 'General operating support', + }); + }); + + it('extracts a person-name recipient via RecipientPersonNm', () => { + const parsed = parse990PfXml(MULTI_GRANT_FIXTURE); + expect(parsed.grantsPaid[1]).toEqual({ + recipientName: 'Jane Q. Public', + recipientCity: 'Concord', + recipientState: 'NH', + amount: 5000, + purpose: 'Scholarship', + }); + }); + + it('coerces a single grant group (fast-xml-parser collapses one-element arrays)', () => { + const parsed = parse990PfXml(SINGLE_GRANT_FIXTURE); + expect(parsed.grantsPaid).toHaveLength(1); + expect(parsed.grantsPaid[0]).toMatchObject({ + recipientName: 'Lone Grantee Org', + amount: 1200, + recipientCity: null, + recipientState: null, + purpose: null, + }); + }); + + it('extracts application submission info into a plain object', () => { + const parsed = parse990PfXml(MULTI_GRANT_FIXTURE); + expect(parsed.applicationInfo).toEqual({ + recipientName: 'See attached list', + formAndInfoAndMaterials: 'Submit a two-page letter of inquiry.', + submissionDeadlines: 'March 1 and September 1 annually.', + restrictionsOnAwards: + 'Grants limited to 501(c)(3) organizations in New Hampshire.', + }); + }); + + it('prefers ReturnHeader/TaxYr for the tax year', () => { + expect(parse990PfXml(MULTI_GRANT_FIXTURE).taxYear).toBe(2026); + expect(parse990PfXml(SINGLE_GRANT_FIXTURE).taxYear).toBe(2025); + }); + + it('falls back to the year in TaxPeriodEndDt when TaxYr is absent', () => { + expect(parse990PfXml(NO_GRANTS_FIXTURE).taxYear).toBe(2024); + }); + + it('returns an empty grants array and null applicationInfo when the group is empty', () => { + const parsed = parse990PfXml(NO_GRANTS_FIXTURE); + expect(parsed.grantsPaid).toEqual([]); + expect(parsed.applicationInfo).toBeNull(); + }); + + it('is defensive against a totally missing IRS990PF/ReturnHeader shape', () => { + const parsed = parse990PfXml( + '', + ); + expect(parsed).toEqual({ + taxYear: null, + applicationInfo: null, + grantsPaid: [], + }); + }); +}); diff --git a/apps/outreach-worker/src/sources/irs-990pf/parse-990pf-xml.ts b/apps/outreach-worker/src/sources/irs-990pf/parse-990pf-xml.ts new file mode 100644 index 0000000..d243cb3 --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/parse-990pf-xml.ts @@ -0,0 +1,156 @@ +/** + * Pure parse of a 990-PF e-file XML document (`Return/ReturnData/IRS990PF`) + * into the fields the funder-precedent index needs: the filing's tax year, + * Part XV application-submission info, and Part XV line 3a grants paid. + * + * Namespace prefixes on these documents vary by filing software vendor, so + * the parser is configured with `removeNSPrefix: true` and every group is + * read defensively — a missing group resolves to an empty array/`null` + * rather than throwing, and single-element repeated groups (which + * `fast-xml-parser` collapses to a bare object) are coerced back to arrays. + */ +import { XMLParser } from 'fast-xml-parser'; + +export interface GrantPaid { + readonly recipientName: string; + readonly recipientCity: string | null; + readonly recipientState: string | null; + readonly amount: number | null; + readonly purpose: string | null; +} + +export interface Parsed990Pf { + readonly taxYear: number | null; + readonly applicationInfo: Record | null; + readonly grantsPaid: GrantPaid[]; +} + +const parser = new XMLParser({ + ignoreAttributes: true, + removeNSPrefix: true, + trimValues: true, + isArray: (_name, jpath) => + jpath.endsWith('GrantOrContributionPdDurYrGrp') || + jpath.endsWith('ApplicationSubmissionInfoGrp'), +}); + +function coerceArray(value: T | T[] | null | undefined): T[] { + if (value == null) return []; + return Array.isArray(value) ? value : [value]; +} + +function textOf(value: unknown): string | null { + if (value == null) return null; + const s = String(value).trim(); + return s === '' ? null : s; +} + +function amountOf(value: unknown): number | null { + if (value == null) return null; + const n = typeof value === 'number' ? value : Number(String(value).trim()); + return Number.isFinite(n) ? Math.round(n) : null; +} + +/** + * `ReturnHeader/TaxYr` when present; otherwise derived from the + * `TaxPeriodEndDt` (`YYYY-MM-DD`) date's calendar year. + */ +function extractTaxYear(header: Record | null): number | null { + if (header == null) return null; + + const taxYr = header.TaxYr; + if (taxYr != null) { + const n = Number(String(taxYr).trim()); + if (Number.isFinite(n)) return n; + } + + const endDt = header.TaxPeriodEndDt; + if (typeof endDt === 'string') { + const match = /^(\d{4})-\d{2}-\d{2}/.exec(endDt.trim()); + if (match != null) return Number(match[1]); + } + + return null; +} + +function extractGrantsPaid(pf: Record | null): GrantPaid[] { + const suppInfo = pf?.SupplementaryInformationGrp as + Record | undefined; + const rows = coerceArray>( + suppInfo?.GrantOrContributionPdDurYrGrp as + Record[] | Record | undefined, + ); + + const grants: GrantPaid[] = []; + for (const row of rows) { + const businessName = row.RecipientBusinessName as + Record | undefined; + const recipientName = + textOf(businessName?.BusinessNameLine1Txt) ?? + textOf(row.RecipientPersonNm); + if (recipientName == null) continue; // no usable recipient identity + + const address = row.RecipientUSAddress as + Record | undefined; + + grants.push({ + recipientName, + recipientCity: textOf(address?.CityNm), + recipientState: textOf(address?.StateAbbreviationCd), + amount: amountOf(row.Amt), + purpose: textOf(row.GrantOrContributionPurposeTxt), + }); + } + return grants; +} + +/** + * Captures whatever Part XV "application submission info" text fields are + * present (form/deadline/address/restriction narrative) into a plain + * object. Filers vary widely in how much of this they fill in, so every + * field is optional and the result is `null` if none are present. + */ +function extractApplicationInfo( + pf: Record | null, +): Record | null { + const suppInfo = pf?.SupplementaryInformationGrp as + Record | undefined; + const raw = + (suppInfo?.ApplicationSubmissionInfoGrp as + Record[] | Record | undefined) ?? + (pf?.ApplicationSubmissionInfoGrp as + Record[] | Record | undefined); + const groups = coerceArray>(raw); + const first = groups[0]; + if (first == null) return null; + + const result: Record = {}; + const recipientName = textOf(first.RecipientNm); + if (recipientName != null) result.recipientName = recipientName; + const formAndInfo = textOf(first.FormAndInfoAndMaterialsTxt); + if (formAndInfo != null) result.formAndInfoAndMaterials = formAndInfo; + const deadlines = textOf(first.SubmissionDeadlinesTxt); + if (deadlines != null) result.submissionDeadlines = deadlines; + const restrictions = textOf(first.RestrictionsOnAwardsTxt); + if (restrictions != null) result.restrictionsOnAwards = restrictions; + + return Object.keys(result).length === 0 ? null : result; +} + +export function parse990PfXml(xml: string): Parsed990Pf { + const parsed = parser.parse(xml) as { + Return?: { + ReturnHeader?: Record; + ReturnData?: { IRS990PF?: Record }; + }; + }; + + const header = parsed.Return?.ReturnHeader ?? null; + const pf = parsed.Return?.ReturnData?.IRS990PF ?? null; + + return { + taxYear: extractTaxYear(header), + applicationInfo: extractApplicationInfo(pf), + grantsPaid: extractGrantsPaid(pf), + }; +} diff --git a/apps/outreach-worker/src/sources/irs-990pf/zip.test.ts b/apps/outreach-worker/src/sources/irs-990pf/zip.test.ts new file mode 100644 index 0000000..2ebfae9 --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/zip.test.ts @@ -0,0 +1,51 @@ +import { strToU8, zipSync } from 'fflate'; +import { describe, expect, it } from 'vitest'; + +import { extractEntries } from './zip.js'; + +function buildFixtureZip(): Uint8Array { + return zipSync({ + '111_public.xml': strToU8('wanted-one'), + '222_public.xml': strToU8('wanted-two'), + '333_public.xml': strToU8('not-wanted'), + 'index.json': strToU8('{}'), + }); +} + +describe('extractEntries', () => { + it('extracts only the wanted entries, keyed by name', () => { + const zip = buildFixtureZip(); + const wanted = new Set(['111_public.xml', '222_public.xml']); + + const entries = extractEntries(zip, wanted); + + expect(entries.size).toBe(2); + expect(new TextDecoder().decode(entries.get('111_public.xml'))).toBe( + 'wanted-one', + ); + expect(new TextDecoder().decode(entries.get('222_public.xml'))).toBe( + 'wanted-two', + ); + }); + + it('does not include entries outside the wanted set', () => { + const zip = buildFixtureZip(); + const entries = extractEntries(zip, new Set(['111_public.xml'])); + + expect(entries.has('333_public.xml')).toBe(false); + expect(entries.has('index.json')).toBe(false); + expect(entries.size).toBe(1); + }); + + it('returns an empty map when none of the wanted names are present', () => { + const zip = buildFixtureZip(); + const entries = extractEntries(zip, new Set(['999_public.xml'])); + expect(entries.size).toBe(0); + }); + + it('returns an empty map for an empty wanted set', () => { + const zip = buildFixtureZip(); + const entries = extractEntries(zip, new Set()); + expect(entries.size).toBe(0); + }); +}); diff --git a/apps/outreach-worker/src/sources/irs-990pf/zip.ts b/apps/outreach-worker/src/sources/irs-990pf/zip.ts new file mode 100644 index 0000000..25ad78e --- /dev/null +++ b/apps/outreach-worker/src/sources/irs-990pf/zip.ts @@ -0,0 +1,28 @@ +/** + * Thin wrapper over `fflate` for selective extraction from a 990-PF batch + * ZIP. Batches run 100-400MB and contain thousands of `{objectId}_public.xml` + * entries; we only ever want the handful matching this run's target object + * ids, so `unzipSync`'s `filter` is used to skip inflating everything else — + * memory stays bounded by the (already-in-memory) zip buffer plus the small + * set of extracted XMLs, not the full uncompressed archive. + */ +import { unzipSync } from 'fflate'; + +/** + * Extracts only the entries in `wantedNames` from `zipBuffer`, keyed by + * their in-archive filename. + */ +export function extractEntries( + zipBuffer: Uint8Array, + wantedNames: ReadonlySet, +): Map { + const files = unzipSync(zipBuffer, { + filter: (file) => wantedNames.has(file.name), + }); + + const result = new Map(); + for (const [name, data] of Object.entries(files)) { + result.set(name, data); + } + return result; +} diff --git a/apps/outreach-worker/src/workflows/ingest-990pf.ts b/apps/outreach-worker/src/workflows/ingest-990pf.ts new file mode 100644 index 0000000..5709861 --- /dev/null +++ b/apps/outreach-worker/src/workflows/ingest-990pf.ts @@ -0,0 +1,693 @@ +/** + * Monthly IRS 990-PF funder-precedent ingestion workflow. + * + * Private foundations rarely publish an open RFP — Grants.gov/PND/NHDOJ + * never see them. Instead this workflow builds a "precedent index": it + * discovers NH-registered private foundations from the IRS Business Master + * File, cross-references the IRS e-file index to find each one's latest + * 990-PF XML filing, parses out grants actually paid to NH nonprofits, and + * synthesizes a `grants` row per foundation with enough NH giving history + * ("this foundation funds orgs like you") — the only signal available for + * funders with no public application process. + * + * Flow: + * 1. BMF discovery — fetch the NH BMF extract, filter to 990-PF filers, + * upsert every one as a `funders` row (refreshes name/city/state/ + * NTEE/assets in place; never clobbers filing-derived fields — see + * `serverUpsertFunder`'s COALESCE logic). + * 2. E-file index — fetch the current + previous year's index CSV(s), + * select the latest 990PF filing per target EIN. + * 3. Skip filter — drop (EIN, filing) pairs whose selected object id + * matches the funder's already-recorded `latestObjectId` (nothing + * changed since the last run). + * 4. Batch processing — group remaining filings by batch ZIP, process the + * batches with the most hits first (capped per run), each batch: fetch + * the ZIP to a temp file, extract only the wanted `{objectId}_public + * .xml` entries, parse each, upsert the funder's filing-derived fields + * and replace its grants-paid rows for that tax year, then delete the + * temp file. + * 5. Synthesis — for funders with enough NH giving history, build and + * upsert a `grants` row summarizing that history as a lead. + * + * Registration follows `ingest-grants.ts` exactly: the scheduled function + * must ALSO be registered as a plain workflow (both registrations + * referencing the same function object), and deps are pulled from a + * module-scope registry populated before `DBOS.launch()` — DBOS serializes + * workflow args, so closures/functions can't cross that boundary. Values + * that DO cross a `DBOS.registerStep` boundary are kept JSON-serializable + * (plain objects/arrays, never `Map`/`Set`) for the same reason. + */ +import { createWriteStream } from 'node:fs'; +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; + +import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk'; +import type { schema } from '@novelpad/outreach-core'; +import { + serverInsertGrants, + serverListFunderLatestObjectIds, + serverListFunderSynthesisData, + serverReplaceFunderGrantsForYear, + serverUpsertFunder, + type FunderSynthesisRow, + type NewGrantInput, +} from '@novelpad/outreach-core/server'; +import type { NodePgDatabase } from 'drizzle-orm/node-postgres'; + +import { + fetchBmfCsv, + parseBmfFoundations, + type BmfFoundation, +} from '#~/sources/irs-990pf/bmf.js'; +import { + fetchIndexCsv, + selectFilings, + type SelectedFiling, +} from '#~/sources/irs-990pf/index-csv.js'; +import { parse990PfXml } from '#~/sources/irs-990pf/parse-990pf-xml.js'; +import { extractEntries } from '#~/sources/irs-990pf/zip.js'; + +export type OutreachDb = NodePgDatabase; + +/** + * A `SelectedFiling` tagged with the e-file index year it was found under + * — the directory the filing's batch ZIP lives in + * (`.../xml/{indexYear}/{batchId}.zip`), which need not equal the filing's + * `taxPeriod` year (see the comment where this is assigned below). + */ +interface IndexedFiling extends SelectedFiling { + readonly indexYear: number; +} + +/** The one state this vertical (and the rest of the outreach engine) targets. */ +const BMF_STATE = 'nh'; +const RECIPIENT_STATE = 'NH'; +/** Minimum in-state grants-paid rows before a foundation is worth synthesizing a lead from. */ +const MIN_STATE_GRANTS = 2; +/** Default batch-ZIP cap per run; overridable for backfills/supervised runs. */ +const DEFAULT_MAX_BATCHES_PER_RUN = 4; + +const BROWSER_HEADERS = { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', +} as const; + +export interface Ingest990PfDeps { + readonly db: OutreachDb; +} + +let registeredDeps: Ingest990PfDeps | null = null; + +export function setIngest990pfDeps(deps: Ingest990PfDeps): void { + registeredDeps = deps; +} + +function getIngest990pfDeps(): Ingest990PfDeps { + if (registeredDeps == null) { + throw new Error( + 'Ingest990PfDeps not registered. Call setIngest990pfDeps() before DBOS.launch().', + ); + } + return registeredDeps; +} + +/** `IRS_990PF_YEARS` env override (comma-sep), else `[currentYear, currentYear - 1]`. */ +function resolveYears(): number[] { + const override = process.env.IRS_990PF_YEARS; + if (override != null && override.trim() !== '') { + const years = override + .split(',') + .map((y) => Number(y.trim())) + .filter((y) => Number.isFinite(y)); + if (years.length > 0) return years; + } + const currentYear = new Date().getUTCFullYear(); + return [currentYear, currentYear - 1]; +} + +function resolveMaxBatches(): number { + const override = process.env.IRS_990PF_MAX_BATCHES_PER_RUN; + if (override != null && override.trim() !== '') { + const n = Number(override.trim()); + if (Number.isFinite(n) && n > 0) return Math.floor(n); + } + return DEFAULT_MAX_BATCHES_PER_RUN; +} + +/** Derives a tax year from an XML parse (preferred) or a `YYYYMM` `TAX_PERIOD` fallback. */ +function resolveTaxYear( + parsedTaxYear: number | null, + taxPeriod: string, +): number | null { + if (parsedTaxYear != null) return parsedTaxYear; + const match = /^(\d{4})\d{2}$/.exec(taxPeriod); + return match != null ? Number(match[1]) : null; +} + +// --------------------------------------------------------------------------- +// Step 1: BMF discovery +// --------------------------------------------------------------------------- + +async function fetchBmf(state: string): Promise { + const csv = await fetchBmfCsv(state); + console.log( + `[ingest-990pf] fetched BMF CSV for state=${state} (${csv.length} bytes)`, + ); + return csv; +} +const fetchBmfStep = DBOS.registerStep(fetchBmf, { + name: 'fetchBmf990Pf', + retriesAllowed: true, + maxAttempts: 3, +}); + +async function upsertFoundation( + db: OutreachDb, + foundation: BmfFoundation, +): Promise { + return serverUpsertFunder(db, { + ein: foundation.ein, + name: foundation.name, + city: foundation.city, + state: foundation.state, + nteeCode: foundation.nteeCode, + totalAssets: foundation.totalAssets, + }); +} +const upsertFoundationStep = DBOS.registerStep(upsertFoundation, { + name: 'upsertBmfFoundation', + retriesAllowed: true, + maxAttempts: 3, +}); + +// --------------------------------------------------------------------------- +// Step 2: e-file index +// --------------------------------------------------------------------------- + +async function fetchIndex(year: number): Promise { + const csv = await fetchIndexCsv(year); + console.log( + `[ingest-990pf] fetched e-file index for year=${year} (${csv.length} bytes)`, + ); + return csv; +} +const fetchIndexStep = DBOS.registerStep(fetchIndex, { + name: 'fetchIrs990PfIndex', + retriesAllowed: true, + maxAttempts: 3, +}); + +// --------------------------------------------------------------------------- +// Step 3: skip filter — funders' already-recorded latestObjectId +// --------------------------------------------------------------------------- + +async function lookupLatestObjectIds( + db: OutreachDb, + eins: string[], +): Promise> { + const rows = await serverListFunderLatestObjectIds(db, eins); + return rows.map((r) => ({ ein: r.ein, latestObjectId: r.latestObjectId })); +} +const lookupLatestObjectIdsStep = DBOS.registerStep(lookupLatestObjectIds, { + name: 'lookupFunderLatestObjectIds', + retriesAllowed: true, + maxAttempts: 3, +}); + +// --------------------------------------------------------------------------- +// Step 4: batch processing +// --------------------------------------------------------------------------- + +/** + * Streams a batch ZIP straight to a temp file (never materializing the full + * 100-400MB response in a JS `ArrayBuffer`), logs its size, and returns the + * temp dir + file path for the caller to read/extract from and clean up. + */ +async function downloadBatchZip( + year: number, + batchId: string, +): Promise<{ dir: string; zipPath: string }> { + const url = `https://apps.irs.gov/pub/epostcard/990/xml/${year}/${batchId}.zip`; + const res = await fetch(url, { headers: BROWSER_HEADERS }); + if (!res.ok) { + throw new Error( + `downloadBatchZip: ${url} responded ${res.status} ${res.statusText}`, + ); + } + if (res.body == null) { + throw new Error(`downloadBatchZip: ${url} returned no response body`); + } + + const dir = await mkdtemp(path.join(tmpdir(), 'irs990pf-')); + const zipPath = path.join(dir, `${batchId}.zip`); + await pipeline( + Readable.fromWeb(res.body as never), + createWriteStream(zipPath), + ); + + const { size } = await stat(zipPath); + console.log( + `[ingest-990pf] downloaded batch ${batchId} (year ${year}): ${(size / (1024 * 1024)).toFixed(1)} MB -> ${zipPath}`, + ); + return { dir, zipPath }; +} + +interface BatchProcessResult { + readonly xmlsParsed: number; + readonly grantsPaidRows: number; + readonly xmlsMissing: number; +} + +/** + * Processes one batch ZIP durably: download, extract only the wanted + * `{objectId}_public.xml` entries, parse + persist each filing, then delete + * the temp file (`finally`, so a mid-loop failure still cleans up). + */ +async function processBatch( + db: OutreachDb, + year: number, + batchId: string, + filings: SelectedFiling[], + foundationByEin: Record, +): Promise { + const { dir, zipPath } = await downloadBatchZip(year, batchId); + + let xmlsParsed = 0; + let grantsPaidRows = 0; + let xmlsMissing = 0; + + try { + const zipBuffer = new Uint8Array(await readFile(zipPath)); + const wantedNames = new Set(filings.map((f) => `${f.objectId}_public.xml`)); + const entries = extractEntries(zipBuffer, wantedNames); + console.log( + `[ingest-990pf] batch ${batchId}: wanted ${wantedNames.size} filing(s), extracted ${entries.size}`, + ); + + const decoder = new TextDecoder('utf-8'); + for (const filing of filings) { + const entryName = `${filing.objectId}_public.xml`; + const bytes = entries.get(entryName); + if (bytes == null) { + xmlsMissing++; + console.warn( + `[ingest-990pf] batch ${batchId}: entry ${entryName} (EIN ${filing.ein}) not found in ZIP; skipping`, + ); + continue; + } + + const foundation = foundationByEin[filing.ein]; + if (foundation == null) { + console.warn( + `[ingest-990pf] batch ${batchId}: no BMF foundation record for EIN ${filing.ein}; skipping`, + ); + continue; + } + + try { + const xml = decoder.decode(bytes); + const parsed = parse990PfXml(xml); + const taxYear = resolveTaxYear(parsed.taxYear, filing.taxPeriod); + if (taxYear == null) { + console.warn( + `[ingest-990pf] batch ${batchId}: could not resolve a tax year for EIN ${filing.ein} (object ${filing.objectId}); skipping`, + ); + continue; + } + + const funderId = await serverUpsertFunder(db, { + ein: filing.ein, + name: foundation.name, + city: foundation.city, + state: foundation.state, + nteeCode: foundation.nteeCode, + totalAssets: foundation.totalAssets, + applicationInfo: parsed.applicationInfo, + latestTaxYear: taxYear, + latestObjectId: filing.objectId, + }); + await serverReplaceFunderGrantsForYear( + db, + funderId, + taxYear, + parsed.grantsPaid, + ); + + xmlsParsed++; + grantsPaidRows += parsed.grantsPaid.length; + } catch (err) { + console.error( + `[ingest-990pf] batch ${batchId}: failed to parse/persist EIN ${filing.ein} (object ${filing.objectId}):`, + err, + ); + } + } + } finally { + await rm(dir, { recursive: true, force: true }).catch((err) => { + console.warn(`[ingest-990pf] failed to clean up temp dir ${dir}:`, err); + }); + } + + return { xmlsParsed, grantsPaidRows, xmlsMissing }; +} +// Two attempts, not three like the lighter-weight steps elsewhere in this +// worker — a batch download is 100-400MB, so retrying a real outage 3x is +// expensive; a transient blip gets one retry and otherwise waits for next +// month's run (batches are re-selected fresh every run, nothing is lost). +const processBatchStep = DBOS.registerStep(processBatch, { + name: 'processIrs990PfBatch', + retriesAllowed: true, + maxAttempts: 2, +}); + +// --------------------------------------------------------------------------- +// Step 5: synthesis +// --------------------------------------------------------------------------- + +function locationClause(city: string | null, state: string | null): string { + if (city != null && state != null) return ` based in ${city}, ${state}`; + if (state != null) return ` based in ${state}`; + if (city != null) return ` based in ${city}`; + return ''; +} + +function applicationInfoSentence(info: unknown): string | null { + if (info == null || typeof info !== 'object') return null; + const rec = info as Record; + const parts: string[] = []; + + if ( + typeof rec.formAndInfoAndMaterials === 'string' && + rec.formAndInfoAndMaterials.trim() !== '' + ) { + parts.push(rec.formAndInfoAndMaterials.trim()); + } + if ( + typeof rec.submissionDeadlines === 'string' && + rec.submissionDeadlines.trim() !== '' + ) { + parts.push(`Deadlines: ${rec.submissionDeadlines.trim()}`); + } + if ( + typeof rec.restrictionsOnAwards === 'string' && + rec.restrictionsOnAwards.trim() !== '' + ) { + parts.push(`Restrictions: ${rec.restrictionsOnAwards.trim()}`); + } + + return parts.length === 0 + ? null + : `Application info from its most recent filing: ${parts.join(' ')}`; +} + +/** Pure: builds a synthesized precedent-lead `grants` row from one funder's aggregated NH giving history. */ +export function buildSynthesizedGrant( + row: FunderSynthesisRow, + now: Date, +): NewGrantInput { + const medianText = + row.medianAmount != null + ? ` typically around $${row.medianAmount.toLocaleString('en-US')}` + : ''; + const purposesText = row.purposes.slice(0, 8).join('; '); + + const sentences: string[] = [ + `Private foundation${locationClause(row.city, row.state)}.`, + `Has made ${row.stateGrantCount} grant${row.stateGrantCount === 1 ? '' : 's'} to New Hampshire organizations in recent filings${medianText}.`, + ]; + if (purposesText.length > 0) { + sentences.push(`Recent grant purposes include: ${purposesText}.`); + } + const appInfo = applicationInfoSentence(row.applicationInfo); + if (appInfo != null) sentences.push(appInfo); + + const awardCeiling = row.maxAmount ?? row.medianAmount ?? null; + const awardFloor = + row.medianAmount != null && row.maxAmount != null + ? Math.min(row.medianAmount, row.maxAmount) + : (row.medianAmount ?? row.maxAmount ?? null); + + return { + sourceUrl: `https://projects.propublica.org/nonprofits/organizations/${row.ein}`, + source: 'irs_990pf', + funderEin: row.ein, + funder: row.name, + title: `${row.name} — grants for New Hampshire nonprofits`, + synopsis: sentences.join(' '), + eligibilityEntityTypes: null, + // `serverListFunderSynthesisData` aggregates grants paid INTO + // recipientState only — we have no visibility into this funder's + // TOTAL giving footprint from this query, so we can't tell whether NH + // is an exclusive restriction or just where it happens to have given + // before. Asserting 'New Hampshire' here would fake a hard geography + // gate off data that doesn't support it; leaving it null keeps the NH + // signal in the precedent subscore instead. (Would need a + // total-grant-count column added to the synthesis query to assert + // this safely — out of scope: core changes weren't part of this task.) + geographicScope: null, + programAreas: null, + awardFloor, + awardCeiling, + expectedAwardsCount: null, + openDate: null, + closeDate: null, + matchRequirement: false, + applicationEffortEstimate: 'unknown', + applicationFormSupported: false, + status: 'open', + lastVerifiedAt: now, + }; +} + +async function synthesizeFunderGrants(db: OutreachDb): Promise { + const rows = await serverListFunderSynthesisData(db, { + recipientState: RECIPIENT_STATE, + minStateGrants: MIN_STATE_GRANTS, + }); + if (rows.length === 0) return 0; + + const now = new Date(); + const grants = rows.map((row) => buildSynthesizedGrant(row, now)); + await serverInsertGrants(db, grants); + return grants.length; +} +const synthesizeFunderGrantsStep = DBOS.registerStep(synthesizeFunderGrants, { + name: 'synthesizeFunderGrants', + retriesAllowed: true, + maxAttempts: 3, +}); + +// --------------------------------------------------------------------------- +// Workflow +// --------------------------------------------------------------------------- + +async function runIngest990Pf(): Promise { + const { db } = getIngest990pfDeps(); + + // 1. BMF discovery. + const bmfCsv = await fetchBmfStep(BMF_STATE); + const foundations = parseBmfFoundations(bmfCsv); + console.log( + `[ingest-990pf] BMF foundations (990-PF filers): ${foundations.length}`, + ); + + const funderIdByEin = new Map(); + const foundationByEin: Record = {}; + for (const foundation of foundations) { + const funderId = await upsertFoundationStep(db, foundation); + funderIdByEin.set(foundation.ein, funderId); + foundationByEin[foundation.ein] = foundation; + } + console.log(`[ingest-990pf] foundations upserted: ${funderIdByEin.size}`); + + if (funderIdByEin.size === 0) { + console.log( + '[ingest-990pf] no 990-PF filers found in BMF this run; skipping filings + synthesis', + ); + return; + } + + // 2. e-file index — current + previous year(s), latest filing per EIN. + // `indexYear` is the directory the filing's index CSV (and therefore its + // batch ZIP) was fetched from — NOT necessarily the filing's tax period + // year (a filing e-filed/processed in 2027 can carry TAX_PERIOD 202612) + // — so it's tracked alongside each filing rather than re-derived later. + const targetEins = new Set(funderIdByEin.keys()); + const years = resolveYears(); + const selectedByEin = new Map(); + + for (const year of years) { + let indexCsv: string; + try { + indexCsv = await fetchIndexStep(year); + } catch (err) { + console.warn( + `[ingest-990pf] e-file index fetch failed for year ${year}; skipping that year:`, + err, + ); + continue; + } + + const filings = selectFilings(indexCsv, targetEins); + console.log( + `[ingest-990pf] year ${year}: ${filings.length} matching 990PF filing(s)`, + ); + + for (const filing of filings) { + const existing = selectedByEin.get(filing.ein); + if ( + existing == null || + Number(filing.taxPeriod) > Number(existing.taxPeriod) + ) { + selectedByEin.set(filing.ein, { ...filing, indexYear: year }); + } + } + } + console.log( + `[ingest-990pf] filings selected across ${years.length} year(s): ${selectedByEin.size}`, + ); + + if (selectedByEin.size > 0) { + // 3. Skip filter. + const latestObjectIdRows = await lookupLatestObjectIdsStep(db, [ + ...selectedByEin.keys(), + ]); + const latestObjectIdByEin = new Map( + latestObjectIdRows.map((r) => [r.ein, r.latestObjectId]), + ); + + const toProcess: IndexedFiling[] = []; + let skippedUnchanged = 0; + for (const filing of selectedByEin.values()) { + const known = latestObjectIdByEin.get(filing.ein); + if (known != null && known === filing.objectId) { + skippedUnchanged++; + continue; + } + toProcess.push(filing); + } + if (skippedUnchanged > 0) { + console.log( + `[ingest-990pf] skipping ${skippedUnchanged} filing(s) already parsed (unchanged latestObjectId)`, + ); + } + + // 4. Group by batch (scoped to the index year the batch ZIP actually + // lives under), most-hits-first, capped. + const batchesByYear = new Map< + string, + { year: number; batchId: string; filings: SelectedFiling[] } + >(); + for (const filing of toProcess) { + const key = `${filing.indexYear}::${filing.batchId}`; + const entry = batchesByYear.get(key) ?? { + year: filing.indexYear, + batchId: filing.batchId, + filings: [], + }; + entry.filings.push(filing); + batchesByYear.set(key, entry); + } + + const sortedBatches = [...batchesByYear.values()].sort( + (a, b) => b.filings.length - a.filings.length, + ); + const maxBatches = resolveMaxBatches(); + const batchesToProcess = sortedBatches.slice(0, maxBatches); + const deferredBatches = sortedBatches.slice(maxBatches); + + if (deferredBatches.length > 0) { + const deferredFilings = deferredBatches.reduce( + (n, b) => n + b.filings.length, + 0, + ); + console.warn( + `[ingest-990pf] batch cap ${maxBatches} reached: deferring ${deferredBatches.length} batch(es) (${deferredFilings} filing(s)) to a future run — ${deferredBatches.map((b) => `${b.batchId}(${b.filings.length})`).join(', ')}`, + ); + } + + let xmlsParsedTotal = 0; + let grantsPaidTotal = 0; + let xmlsMissingTotal = 0; + for (const batch of batchesToProcess) { + try { + const result = await processBatchStep( + db, + batch.year, + batch.batchId, + batch.filings, + foundationByEin, + ); + xmlsParsedTotal += result.xmlsParsed; + grantsPaidTotal += result.grantsPaidRows; + xmlsMissingTotal += result.xmlsMissing; + } catch (err) { + console.error(`[ingest-990pf] batch ${batch.batchId} failed:`, err); + } + } + console.log( + `[ingest-990pf] batches processed=${batchesToProcess.length} deferred=${deferredBatches.length} xmlsParsed=${xmlsParsedTotal} xmlsMissing=${xmlsMissingTotal} grantsPaidRows=${grantsPaidTotal}`, + ); + } else { + console.log( + '[ingest-990pf] no filings selected this run; skipping batch processing', + ); + } + + // 5. Synthesis. + const synthesized = await synthesizeFunderGrantsStep(db); + console.log( + `[ingest-990pf] synthesized funder-precedent grants upserted: ${synthesized}`, + ); +} + +const g = globalThis as unknown as { + __outreachIngest990PfRegistered?: boolean; + __outreachIngest990PfHandle?: ( + scheduledTime: Date, + startedAt: Date, + ) => Promise; +}; + +if (!g.__outreachIngest990PfRegistered) { + g.__outreachIngest990PfRegistered = true; + + const ingest990pf = async (_scheduledTime: Date, _startedAt: Date) => { + try { + await runIngest990Pf(); + } catch (err) { + console.error('[ingest-990pf] pass failed:', err); + throw err; + } + }; + + // Must be registered as BOTH a workflow and a scheduled function, + // referencing the same function object — see module doc comment. + g.__outreachIngest990PfHandle = DBOS.registerWorkflow(ingest990pf, { + name: 'ingest990pf', + }); + DBOS.registerScheduled(ingest990pf, { + crontab: '0 6 2 * *', + name: 'ingest990pf', + mode: SchedulerMode.ExactlyOncePerInterval, + }); +} + +/** + * Starts one durable run of this workflow immediately through DBOS — + * the exact production path (workflow + checkpointed steps), used by + * `run-once.ts` for supervised/manual passes. Requires deps injected and + * `DBOS.launch()` completed. + */ +export function runIngest990PfNow(): Promise { + const handle = g.__outreachIngest990PfHandle; + if (handle == null) { + throw new Error( + 'ingest990pf is not registered; was this module imported before DBOS.launch()?', + ); + } + return handle(new Date(), new Date()); +} diff --git a/apps/outreach-worker/src/workflows/match-grants.ts b/apps/outreach-worker/src/workflows/match-grants.ts index 703a5e1..8d2a5ba 100644 --- a/apps/outreach-worker/src/workflows/match-grants.ts +++ b/apps/outreach-worker/src/workflows/match-grants.ts @@ -108,8 +108,10 @@ const ensureOrgEmbeddingStep = DBOS.registerStep(ensureOrgEmbedding, { async function retrieveGrants( db: OutreachDb, embedding: number[], + orgState: string, ): Promise { return serverListEligibleGrantsForOrg(db, embedding, { + orgState, minDaysToDeadline: MIN_DAYS_TO_DEADLINE, minAwardCeiling: MIN_AWARD_CEILING, limit: GRANTS_PER_ORG, @@ -121,6 +123,14 @@ const retrieveGrantsStep = DBOS.registerStep(retrieveGrants, { maxAttempts: 3, }); +/** Case/punctuation/suffix-insensitive equality for self-match detection. */ +function normalizeSelfMatchName(name: string): string { + return name + .toUpperCase() + .replace(/\b(INC|INCORPORATED|TTEE|TRUSTEE|FUND|FOUNDATION|CHARITABLE|TRUST)\b/g, '') + .replace(/[^A-Z0-9]/g, ''); +} + async function scoreAndStoreOrgMatches( db: OutreachDb, org: MatchCandidateOrg, @@ -131,6 +141,22 @@ async function scoreAndStoreOrgMatches( let gated = 0; for (const grant of grants) { + // A foundation's synthesized grant must never match the foundation's + // own org row ("we found you $50K — from yourself"). EIN when both + // sides have one; normalized-name fallback because many NHDOJ org rows + // haven't resolved an EIN yet — a self-match is worse than a missed + // match here. + const isSelfByEin = + grant.funderEin != null && + org.ein != null && + grant.funderEin === org.ein; + const isSelfByName = + grant.funderEin != null && + normalizeSelfMatchName(grant.funder) === normalizeSelfMatchName(org.name); + if (isSelfByEin || isSelfByName) { + gated++; + continue; + } const gates = evaluateHardGates( { entityType: ASSUMED_ENTITY_TYPE, state: org.state }, { @@ -153,6 +179,7 @@ async function scoreAndStoreOrgMatches( const scored = scoreMatch({ similarity: grant.similarity, + funderStateGrantCount: grant.funderStateGrantCount, orgTotalRevenue: org.totalRevenue, awardCeiling: grant.awardCeiling, geographicScope: grant.geographicScope, @@ -173,7 +200,7 @@ async function scoreAndStoreOrgMatches( gateFailures: gates.failures, ignoredGates: [...IGNORED_GATES], scoredAt: now.toISOString(), - scoringVersion: 'v1-no-precedent', + scoringVersion: 'v2-state-precedent', }, }); stored++; @@ -203,7 +230,7 @@ async function runMatchGrants(): Promise { for (const org of orgs) { try { const embedding = await ensureOrgEmbeddingStep(db, org); - const grants = await retrieveGrantsStep(db, embedding); + const grants = await retrieveGrantsStep(db, embedding, org.state); const { stored, gated } = await scoreAndStoreOrgMatchesStep( db, org, diff --git a/docs/features/ingestion.md b/docs/features/ingestion.md index 2292e2b..fc0d37c 100644 --- a/docs/features/ingestion.md +++ b/docs/features/ingestion.md @@ -6,14 +6,15 @@ Grant upserts key on `grants.source_url`; org registry upserts key on case-insen ## Schedules -| Workflow | Cron (UTC) | Source | -|---|---|---| -| `ingestGrants` | `0 3 * * *` | Grants.gov Search2 | -| `ingestPndRss` | `30 3 * * *` | Philanthropy News Digest RFP feed | -| `expireGrants` | `0 * * * *` | (sweep: `status='expired'` past `close_date`) | -| `enrichOrgs` | `0 5 * * *` | ProPublica Nonprofit Explorer | -| `embedGrants` | `15 4 * * *` | gemini-embedding-001 over open-grant synopses → `grants.synopsis_embedding` (first paid AI call; ~pennies/batch) | -| `ingestNhdojOrgs` | `0 4 1 * *` | NHDOJ Charitable Trusts registry PDF | +| Workflow | Cron (UTC) | Source | +| ----------------- | ------------ | ---------------------------------------------------------------------------------------------------------------- | +| `ingestGrants` | `0 3 * * *` | Grants.gov Search2 | +| `ingestPndRss` | `30 3 * * *` | Philanthropy News Digest RFP feed | +| `expireGrants` | `0 * * * *` | (sweep: `status='expired'` past `close_date`) | +| `enrichOrgs` | `0 5 * * *` | ProPublica Nonprofit Explorer | +| `embedGrants` | `15 4 * * *` | gemini-embedding-001 over open-grant synopses → `grants.synopsis_embedding` (first paid AI call; ~pennies/batch) | +| `ingestNhdojOrgs` | `0 4 1 * *` | NHDOJ Charitable Trusts registry PDF | +| `ingest990pf` | `0 6 2 * *` | IRS BMF + e-file index + batch ZIPs → funders/funder_grants → synthesized foundation grants | ## Sources @@ -26,16 +27,19 @@ Grant upserts key on `grants.source_url`; org registry upserts key on case-insen **Cadence**: Nightly, `0 3 * * *` (03:00 UTC), registered as a DBOS scheduled workflow (`ingestGrants`, `ExactlyOncePerInterval`). **API**: Public Grants.gov Search2 API, no key required. + - `POST https://api.grants.gov/v1/api/search2` — enumerates opportunities, filtered to `oppStatuses: 'posted'` and nonprofit eligibility codes `12|13` (501(c)(3) and non-501(c)(3) nonprofits). Paginated via `rows`/`startRecordNum`. - `POST https://api.grants.gov/v1/api/fetchOpportunity` — full detail (`synopsis`: description, applicant types, award amounts, response date) for a single opportunity id. - Both endpoints wrap responses in an `{ errorcode, msg, data }` envelope. Any non-2xx HTTP status or `errorcode !== 0` throws immediately with a descriptive message — the client never silently drops or swallows an upstream failure. **Caps** (per nightly run): + - Search enumeration: up to **1000** hits (`SEARCH_HIT_CAP`). Count logged via `console.log`. - Detail fetch: up to **200** opportunities (`DETAIL_FETCH_CAP`), each a separate `fetchOpportunity` call with a **~250ms** politeness delay between requests. When the search result set exceeds the cap, the excess is dropped for that run (picked up on a later run) and a `console.warn` states exactly how many opportunities were skipped — never a silent truncation. - Upsert: batched in groups of **100** via `serverInsertGrants`. **Normalization rules** (`normalizeGrantsGovOpportunity`, pure — no I/O): + - `sourceUrl` (the upsert key): `https://www.grants.gov/search-results-detail/{opportunityId}` — stable across re-crawls. - `funder`: `agencyDetails.agencyName` → `synopsis.agencyName` → search hit's `agency` → an agency code fallback → literal `'Unknown federal agency'` if nothing is present. - `synopsis`: HTML-stripped from `synopsis.synopsisDesc` (block tags become newlines, common entities decoded); `null` if empty after stripping. @@ -71,6 +75,7 @@ Nightly ingestion of the Philanthropy News Digest "RFPs" RSS feed (`https://phil Daily scheduled workflow (`enrichOrgs`, cron `0 5 * * *`) that fills in IRS-derived fields — EIN, NTEE code, most-recent-filing total revenue, fiscal year-end month — for orgs discovered by the other ingestion sources but never resolved against the IRS. **Flow:** + 1. `serverListOrgsNeedingEnrichment(db, { limit: 200 })` — orgs with no EIN and no revenue on file, oldest-`updatedAt` first. 2. For each org, sequentially (never `forEach`+async — the client's politeness delay depends on awaiting each call before starting the next): - `searchOrganizations(org.name, org.state)` against Nonprofit Explorer's `search.json`. @@ -79,7 +84,7 @@ Daily scheduled workflow (`enrichOrgs`, cron `0 5 * * *`) that fills in IRS-deri - On match: `getOrganization(ein)` for filing history, `extractEnrichment(detail)`, then `serverEnrichOrg(db, org.id, enrichment)`. 3. Per-org failures are caught, logged, and counted — one bad org doesn't kill the batch. If failures exceed 20% of the attempted batch, the workflow rethrows (systemic-failure signal for DBOS retry/alerting) after logging attempted/resolved/unresolved/failed counts. -**Matching (`src/sources/propublica/match.ts`):** `normalizeOrgName` lowercases, strips punctuation (apostrophes drop silently, other punctuation becomes a separator), strips legal-suffix/article noise tokens (`inc`, `corp`, `the`, `of`, `nh`, the phrase `new hampshire`), and collapses whitespace. `pickBestMatch` prefers an exact normalized-name match, falls back to token-set Jaccard similarity ≥ 0.8, and disqualifies any candidate whose known city differs from the target's known city — on *both* paths, since same-legal-name-different-town is exactly the ambiguous case worth refusing rather than guessing. Ties break by city match, then shortest Levenshtein distance on the normalized name. A wrong EIN silently poisons downstream revenue/ICP-band data with no cheap way to detect it later, so every ambiguous case resolves to `null` (org stays in next run's backlog) instead of a best-effort guess. +**Matching (`src/sources/propublica/match.ts`):** `normalizeOrgName` lowercases, strips punctuation (apostrophes drop silently, other punctuation becomes a separator), strips legal-suffix/article noise tokens (`inc`, `corp`, `the`, `of`, `nh`, the phrase `new hampshire`), and collapses whitespace. `pickBestMatch` prefers an exact normalized-name match, falls back to token-set Jaccard similarity ≥ 0.8, and disqualifies any candidate whose known city differs from the target's known city — on _both_ paths, since same-legal-name-different-town is exactly the ambiguous case worth refusing rather than guessing. Ties break by city match, then shortest Levenshtein distance on the normalized name. A wrong EIN silently poisons downstream revenue/ICP-band data with no cheap way to detect it later, so every ambiguous case resolves to `null` (org stays in next run's backlog) instead of a best-effort guess. **Extraction (`src/sources/propublica/extract.ts`):** `extractEnrichment` zero-pads the numeric EIN to 9 digits, passes through `ntee_code` (null-safe), and — from `filings_with_data` — picks the filing with the highest `tax_prd_yr` for `totalRevenue` and derives `fiscalYearEnd` as the zero-padded `MM` from that filing's `tax_prd` (`YYYYMM`, e.g. `202306` → `'06'`). Empty filing history yields `totalRevenue: null, fiscalYearEnd: null`. @@ -98,11 +103,39 @@ Monthly re-scan of the NH Department of Justice Charitable Trusts Unit's registr - **Config gap ≠ outage**: if `NHDOJ_REGISTRY_PDF_URL` is unset, the workflow logs a `console.warn` and returns without throwing. NHDOJ has no stable URL for the registry PDF — it changes whenever they republish — so a hard failure here would page on-call for a config gap rather than a real problem. - **Not yet wired into `apps/outreach-worker/src/main.ts`** — the module registers itself as a side effect of being imported (per the pattern above), but `main.ts` needs an explicit import (for registration-before-launch ordering) plus a `setIngestNhdojOrgsDeps({ db })` call before `DBOS.launch()`: ```ts - import { setIngestNhdojOrgsDeps } from './workflows/ingest-nhdoj-orgs.js'; + import { setIngestNhdojOrgsDeps } from "./workflows/ingest-nhdoj-orgs.js"; // ... setIngestNhdojOrgsDeps({ db }); ``` +### IRS 990-PF funder precedent + +Private foundations almost never post an open RFP — Grants.gov/PND/NHDOJ never see them. This vertical instead builds a "precedent index": it discovers NH-registered private foundations from the IRS Business Master File (BMF), cross-references the IRS e-file index to find each one's latest 990-PF filing, parses grants actually paid to NH nonprofits out of that filing's XML, and — for foundations with enough NH giving history — synthesizes a `grants` row summarizing that history as a lead ("this foundation funds orgs like you"). Unlike every other source in this doc, it upserts into **both** `funders`/`funder_grants` (the raw precedent data) and `grants` (the synthesized lead), keyed by `grants.source_url` like the rest. + +**Source**: `apps/outreach-worker/src/sources/irs-990pf/` — `bmf.ts` (BMF fetch + parse), `index-csv.ts` (e-file index fetch + filter), `parse-990pf-xml.ts` (990-PF XML → grants paid + application info), `zip.ts` (batch-ZIP selective extraction). Wired into `apps/outreach-worker/src/workflows/ingest-990pf.ts`. + +**Cadence**: Monthly, `0 6 2 * *` (06:00 UTC on the 2nd), registered as a DBOS scheduled workflow (`ingest990pf`, `ExactlyOncePerInterval`). + +**Data sources**, all Akamai-fronted (same browser-header workaround as `fetchNhdojRegistryPdf`): + +1. `https://www.irs.gov/pub/irs-soi/eo_nh.csv` — BMF state extract (~1.6MB). `PF_FILING_REQ_CD === '1'` identifies 990-PF filers. +2. `https://apps.irs.gov/pub/epostcard/990/xml/{year}/index_{year}.csv` — e-file index (~28MB). Filtered in a manual line-by-line scan (no full-file line array, no giant intermediate row array) to `RETURN_TYPE === '990PF'` rows whose EIN is in the BMF-discovered target set, keeping only the latest `TAX_PERIOD` per EIN. +3. `https://apps.irs.gov/pub/epostcard/990/xml/{year}/{XML_BATCH_ID}.zip` — batch ZIPs (100-400MB each), containing `{OBJECT_ID}_public.xml` per filing. Streamed straight to a temp file (`os.tmpdir()`, never buffered whole in the JS heap during download), then read back once and selectively extracted via `fflate`'s `unzipSync({ filter })` — only the wanted entries are inflated. The temp file is deleted after each batch (`finally`, so a mid-batch failure still cleans up). + +**Caps** (per monthly run): + +- Batches processed: up to **4** (`IRS_990PF_MAX_BATCHES_PER_RUN`), the batches with the most target-EIN hits first. Deferred batches are logged by id + hit count via `console.warn` — never a silent truncation. No `processed_batches` checkpoint is needed: re-parsing is idempotent (`serverReplaceFunderGrantsForYear` replaces per tax year) and the target object-id set shrinks on its own as funders' `latestTaxYear`/`latestObjectId` advance. +- Skip filter: an (EIN, filing) pair is skipped entirely — no download credited against it — when the e-file index's selected `OBJECT_ID` for that EIN already matches the funder's recorded `latestObjectId` (`serverListFunderLatestObjectIds`, `packages/outreach-core/src/funders/queries/list-funder-latest-object-ids.server.ts`), i.e. nothing changed since the last run. +- Years scanned: current + previous (`IRS_990PF_YEARS`, comma-sep override) — a filing's e-file index year need not equal its `TAX_PERIOD` year, so each selected filing tracks the actual index year its batch ZIP lives under (`IndexedFiling.indexYear` in the workflow), not a value re-derived from `TAX_PERIOD`. + +**XML parsing** (`parse990PfXml`, pure — no I/O): `removeNSPrefix: true` handles filing-software-dependent namespace prefixes. Grants paid (`SupplementaryInformationGrp/GrantOrContributionPdDurYrGrp[]`) resolve the recipient name from `RecipientBusinessName/BusinessNameLine1Txt`, falling back to `RecipientPersonNm`; amounts round to whole dollars; a row with neither name field is dropped (no usable recipient identity). Application info (`ApplicationSubmissionInfoGrp`) captures whatever of `RecipientNm`/`FormAndInfoAndMaterialsTxt`/`SubmissionDeadlinesTxt`/`RestrictionsOnAwardsTxt` is present into a plain object, `null` if none are. Tax year prefers `ReturnHeader/TaxYr`, falling back to the calendar year of `TaxPeriodEndDt`, falling back (in the workflow, not the pure parser) to the filing's `TAX_PERIOD` (`YYYYMM`) when the XML has neither. Every group defaults to empty/`null` on absence rather than throwing — a missing Part XV section is normal for a foundation with no formal application process. + +**Synthesis** (`buildSynthesizedGrant`, pure): for every funder with ≥2 grants paid into NH (`serverListFunderSynthesisData`), builds one `grants` row: `source: 'irs_990pf'`, `funderEin` set, `sourceUrl` the funder's ProPublica Nonprofit Explorer page (stable, human-followable, and distinct from the funder's own `funders.ein` upsert key so re-running never collides with a same-funder public-RFP row). `synopsis` is composed prose (location, in-state grant count + typical/median amount, up to 8 recent grant purposes, application-info sentence when present). `awardCeiling`/`awardFloor` derive from the aggregated median/max in-state amounts. `geographicScope` is deliberately left `null` — the synthesis query only sees grants paid _into_ NH, not the funder's total giving footprint, so there's no way to tell from this data whether NH is an exclusive restriction; asserting it would fake a hard geography gate the data doesn't support. `closeDate: null` (rolling), `applicationEffortEstimate: 'unknown'`, `status: 'open'`. + +**Workflow** (`ingest-990pf.ts`): same registration pattern as `ingest-grants.ts` (module-scope deps registry via `setIngest990pfDeps`, dual workflow+scheduled registration, `globalThis` guard, `runIngest990PfNow()` accessor for `run-once.ts`). Steps: fetch BMF (3 retries) → upsert every parsed foundation as a `funders` row (per-row step, same pattern as `ingest-nhdoj-orgs.ts`) → fetch each target year's e-file index (3 retries, per-year) → pure `selectFilings` → skip-filter against recorded `latestObjectId` → group into batches, sort by hit count, cap → per batch (2 retries — a 100-400MB download is expensive to retry 3x): download, extract, parse, `serverUpsertFunder` (filing-derived fields) + `serverReplaceFunderGrantsForYear` per filing → synthesis step. Every stage logs its counts via `console.log`/`console.warn`. + +**Core addition**: `packages/outreach-core/src/funders/queries/list-funder-latest-object-ids.server.ts` (`serverListFunderLatestObjectIds`) — the one core addition this vertical needed, following the existing action/query barrel pattern; everything else in `packages/outreach-core/src/funders/` (the `funders`/`funder_grants` schema, `serverUpsertFunder`, `serverReplaceFunderGrantsForYear`, `serverListFunderSynthesisData`) pre-existed this vertical. + ## First-run field findings (2026-07-16) - **NHDOJ registry**: 8-column layout (`Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due`), single-letter statuses (G/X/S legend), includes out-of-state charities registered to solicit in NH. Parsed 13,709 registrants → 13,632 orgs (6,266 NH). `Reg. No.` is the stable upsert key (`orgs.registration_number`). mm.nh.gov sits behind Akamai: bare curl gets 403; Node fetch with browser-like headers passes (client sends them). `NHDOJ_REGISTRY_PDF_PATH` overrides the URL for supervised runs. diff --git a/docs/features/scoring.md b/docs/features/scoring.md index 74e4520..fb8ddc9 100644 --- a/docs/features/scoring.md +++ b/docs/features/scoring.md @@ -7,9 +7,17 @@ Nightly `matchGrants` workflow (05:15 UTC, after embeddings) — the plan's "SQL 1. **Profile embedding** — v0 stub: NTEE-derived mission text (`buildOrgMissionText`) embedded as `RETRIEVAL_QUERY`, stored in `org_profiles` at confidence 0.2. The Stage 4 research profiler upgrades the row in place; this workflow doesn't change. 2. **Retrieval** — `serverListEligibleGrantsForOrg`: one SQL statement enforcing the cheap hard gates (status open, embedded, deadline ≥ 21 days, ceiling ≥ $10K) with pgvector cosine ranking; top 50 per org. 3. **Remaining gates in TS** — entity eligibility (`entryAdmitsEntity`: conservative pattern matching over Grants.gov applicantTypes prose; ambiguous entries do NOT admit) and geography (word-boundary state code + full state name). `application_form_supported` is **deliberately ignored** for pass/fail (2026-07-16 manual-first decision — draftability verified by hand for top leads); its failure still lands in `rationale.gateFailures`. Failed pairs are not stored. -4. **Deterministic subscores** (`scoreMatch`, pure, tested): mission fit 30 (similarity 0.45–0.75 → 0–30) · capacity 15 (award 10–75% of revenue = sweet spot) · competition 15 (state-restricted ≫ national) · effort 10 · runway 5 (3–10 weeks ideal). **Funder precedent (25) not yet awarded** — achievable max is 75 until the 990-PF index lands; `subscores` jsonb keeps the full breakdown for reweighting. Easy win = total ≥ 50. +4. **Deterministic subscores** (`scoreMatch`, pure, tested): mission fit 30 (similarity 0.45–0.75 → 0–30) · capacity 15 (award 10–75% of revenue = sweet spot) · competition 15 (state-restricted ≫ national) · effort 10 · runway 5 (3–10 weeks ideal). **Funder precedent (25pts, live)**: tiers of repeated giving into the org's state from the 990-PF index (1→8, 3→15, 5→20, 10→25); federal/no-data funders score 0 — absence of evidence ranks below presence. Easy win = total ≥ 65 AND precedent ≥ 12 (the plan's precedent floor). `subscores` jsonb keeps the full breakdown for reweighting. 5. **Upsert + hero** — pair-keyed upsert that never touches review fields (a human's reject stands even when scores move); `serverAssignHeroMatch` marks the org's top non-rejected gate-passing match. ## First live run (2026-07-16) 64 orgs × top-50 grants → 3,200 matches, 0 gate failures (corpus was pre-filtered to nonprofit-eligible, federal = geography-unrestricted), **0 easy wins, max 39/75**. That's the system being honest: the current corpus is 200 NIH-dominated federal research grants — wrong pond for $100K–$5M NH service nonprofits (similarity ceiling ~0.58). The engine's next real gains are corpus-side: NH state agency sources, 990-PF foundation ingestion, full Grants.gov detail backlog, real effort estimates. + + +## v2 (2026-07-16, same day): 990-PF precedent + lead-quality gates + +- `ingest990pf` (monthly, `0 6 2 * *`): IRS BMF `eo_nh.csv` discovers NH private foundations (`PF_FILING_REQ_CD=1`) → e-file index CSVs select their latest 990-PF filings → batch ZIPs (capped 4/run, most-hits-first, deferred logged) → grants-paid rows into `funder_grants` → funders with ≥2 NH grants synthesize a rolling `grants` row (source `irs_990pf`, `funder_ein` set, null close date) that flows through embed + match like any RFP. +- Rolling (null) deadlines now PASS the runway gate and score 2/5 runway. +- Candidate orgs exclude NTEE `T*` grantmakers, and matches self-gate by funder EIN plus normalized-name fallback — the first precedent run's top "leads" were foundations matched to themselves (NHDOJ registers grantmakers as charities; several lack resolved EINs). +- First full run: 747 NH foundations, 21 batches (~6.5GB processed, 1 deferred), 2,766+ grants-paid rows, 123 synthesized foundation grants → **89 easy wins across 27 orgs**, top hero 69/100 with real matches like AIDS Response-Seacoast → Foundation for Seacoast Health. Coverage grows nightly as enrichment drains the org backlog (56 candidate orgs of ~6.2K NH registrants so far). diff --git a/packages/outreach-core/drizzle/server/1784236406_funder-precedent-index.sql b/packages/outreach-core/drizzle/server/1784236406_funder-precedent-index.sql new file mode 100644 index 0000000..96b5e57 --- /dev/null +++ b/packages/outreach-core/drizzle/server/1784236406_funder-precedent-index.sql @@ -0,0 +1,35 @@ +CREATE TABLE "funder_grants" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "funder_id" uuid NOT NULL, + "recipient_name" text NOT NULL, + "recipient_city" text, + "recipient_state" text, + "amount" integer, + "purpose" text, + "tax_year" integer NOT NULL, + "created_at" timestamp with time zone DEFAULT now() +); +--> statement-breakpoint +CREATE TABLE "funders" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "ein" text NOT NULL, + "name" text NOT NULL, + "city" text, + "state" text, + "ntee_code" text, + "total_assets" bigint, + "application_info" jsonb, + "latest_tax_year" integer, + "latest_object_id" text, + "created_at" timestamp with time zone DEFAULT now(), + "updated_at" timestamp with time zone DEFAULT now() +); +--> statement-breakpoint +ALTER TABLE "grants" ADD COLUMN "funder_ein" text;--> statement-breakpoint +ALTER TABLE "funder_grants" ADD CONSTRAINT "funder_grants_funder_id_funders_id_fk" FOREIGN KEY ("funder_id") REFERENCES "public"."funders"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "idx_funder_grants_funder" ON "funder_grants" USING btree ("funder_id");--> statement-breakpoint +CREATE INDEX "idx_funder_grants_funder_year" ON "funder_grants" USING btree ("funder_id","tax_year");--> statement-breakpoint +CREATE INDEX "idx_funder_grants_recipient_state" ON "funder_grants" USING btree ("recipient_state");--> statement-breakpoint +CREATE UNIQUE INDEX "idx_funders_ein" ON "funders" USING btree ("ein");--> statement-breakpoint +CREATE INDEX "idx_funders_state" ON "funders" USING btree ("state");--> statement-breakpoint +CREATE INDEX "idx_grants_funder_ein" ON "grants" USING btree ("funder_ein"); \ No newline at end of file diff --git a/packages/outreach-core/drizzle/server/meta/1784236406_snapshot.json b/packages/outreach-core/drizzle/server/meta/1784236406_snapshot.json new file mode 100644 index 0000000..909d100 --- /dev/null +++ b/packages/outreach-core/drizzle/server/meta/1784236406_snapshot.json @@ -0,0 +1,1460 @@ +{ + "id": "d30ecf3d-7041-41cc-a4e7-7d70f38b3218", + "prevId": "4e00eb42-105f-461a-abd0-f7ff237267cf", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.contacts": { + "name": "contacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_status": { + "name": "email_status", + "type": "email_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unverified'" + }, + "source_provider": { + "name": "source_provider", + "type": "contact_source_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "contact_priority", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'generic'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_contacts_org": { + "name": "idx_contacts_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_contacts_email": { + "name": "idx_contacts_email", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "contacts_org_id_orgs_id_fk": { + "name": "contacts_org_id_orgs_id_fk", + "tableFrom": "contacts", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.funder_grants": { + "name": "funder_grants", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "funder_id": { + "name": "funder_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_city": { + "name": "recipient_city", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_state": { + "name": "recipient_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tax_year": { + "name": "tax_year", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_funder_grants_funder": { + "name": "idx_funder_grants_funder", + "columns": [ + { + "expression": "funder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_funder_grants_funder_year": { + "name": "idx_funder_grants_funder_year", + "columns": [ + { + "expression": "funder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tax_year", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_funder_grants_recipient_state": { + "name": "idx_funder_grants_recipient_state", + "columns": [ + { + "expression": "recipient_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "funder_grants_funder_id_funders_id_fk": { + "name": "funder_grants_funder_id_funders_id_fk", + "tableFrom": "funder_grants", + "tableTo": "funders", + "columnsFrom": [ + "funder_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.funders": { + "name": "funders", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "ein": { + "name": "ein", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "city": { + "name": "city", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ntee_code": { + "name": "ntee_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "total_assets": { + "name": "total_assets", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "application_info": { + "name": "application_info", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "latest_tax_year": { + "name": "latest_tax_year", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "latest_object_id": { + "name": "latest_object_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_funders_ein": { + "name": "idx_funders_ein", + "columns": [ + { + "expression": "ein", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_funders_state": { + "name": "idx_funders_state", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grants": { + "name": "grants", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "funder": { + "name": "funder", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "synopsis": { + "name": "synopsis", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "eligibility_entity_types": { + "name": "eligibility_entity_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "geographic_scope": { + "name": "geographic_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "program_areas": { + "name": "program_areas", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "award_floor": { + "name": "award_floor", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "award_ceiling": { + "name": "award_ceiling", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "expected_awards_count": { + "name": "expected_awards_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "open_date": { + "name": "open_date", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_date": { + "name": "close_date", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "match_requirement": { + "name": "match_requirement", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "application_effort_estimate": { + "name": "application_effort_estimate", + "type": "application_effort_estimate", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "application_form_supported": { + "name": "application_form_supported", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "grant_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "funder_ein": { + "name": "funder_ein", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "grant_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "synopsis_embedding": { + "name": "synopsis_embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "last_verified_at": { + "name": "last_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_grants_source_url": { + "name": "idx_grants_source_url", + "columns": [ + { + "expression": "source_url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_grants_status": { + "name": "idx_grants_status", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_grants_close_date": { + "name": "idx_grants_close_date", + "columns": [ + { + "expression": "close_date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_grants_source": { + "name": "idx_grants_source", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_grants_funder_ein": { + "name": "idx_grants_funder_ein", + "columns": [ + { + "expression": "funder_ein", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "grants_synopsis_embedding_idx": { + "name": "grants_synopsis_embedding_idx", + "columns": [ + { + "expression": "synopsis_embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.matches": { + "name": "matches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "total_score": { + "name": "total_score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "subscores": { + "name": "subscores", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hard_gates_passed": { + "name": "hard_gates_passed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "easy_win": { + "name": "easy_win", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "rationale": { + "name": "rationale", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "review_status": { + "name": "review_status", + "type": "match_review_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "reject_reason": { + "name": "reject_reason", + "type": "match_reject_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "is_hero": { + "name": "is_hero", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_matches_org_grant": { + "name": "idx_matches_org_grant", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_matches_org": { + "name": "idx_matches_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_matches_grant": { + "name": "idx_matches_grant", + "columns": [ + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_matches_review_status": { + "name": "idx_matches_review_status", + "columns": [ + { + "expression": "review_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "matches_org_id_orgs_id_fk": { + "name": "matches_org_id_orgs_id_fk", + "tableFrom": "matches", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "matches_grant_id_grants_id_fk": { + "name": "matches_grant_id_grants_id_fk", + "tableFrom": "matches", + "tableTo": "grants", + "columnsFrom": [ + "grant_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_profiles": { + "name": "org_profiles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "mission_statement": { + "name": "mission_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "programs": { + "name": "programs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "service_geography": { + "name": "service_geography", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recent_news": { + "name": "recent_news", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "known_funders": { + "name": "known_funders", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "staff": { + "name": "staff", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "budget_band": { + "name": "budget_band", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sources": { + "name": "sources", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "profile_embedding": { + "name": "profile_embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_org_profiles_org_unique": { + "name": "idx_org_profiles_org_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_org_profiles_org": { + "name": "idx_org_profiles_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_profiles_embedding_idx": { + "name": "org_profiles_embedding_idx", + "columns": [ + { + "expression": "profile_embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": { + "org_profiles_org_id_orgs_id_fk": { + "name": "org_profiles_org_id_orgs_id_fk", + "tableFrom": "org_profiles", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.orgs": { + "name": "orgs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "city": { + "name": "city", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'NH'" + }, + "ein": { + "name": "ein", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ntee_code": { + "name": "ntee_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "total_revenue": { + "name": "total_revenue", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "fiscal_year_end": { + "name": "fiscal_year_end", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_number": { + "name": "registration_number", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_status": { + "name": "registration_status", + "type": "registration_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "icp_band": { + "name": "icp_band", + "type": "icp_band", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "source_registry": { + "name": "source_registry", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_orgs_ein": { + "name": "idx_orgs_ein", + "columns": [ + { + "expression": "ein", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"orgs\".\"ein\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_orgs_registry_reg_no": { + "name": "idx_orgs_registry_reg_no", + "columns": [ + { + "expression": "source_registry", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"orgs\".\"source_registry\" IS NOT NULL AND \"orgs\".\"registration_number\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_orgs_icp_band": { + "name": "idx_orgs_icp_band", + "columns": [ + { + "expression": "icp_band", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_orgs_state": { + "name": "idx_orgs_state", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pipeline_events": { + "name": "pipeline_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "contact_id": { + "name": "contact_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "match_id": { + "name": "match_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "pipeline_event_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_pipeline_events_org": { + "name": "idx_pipeline_events_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_pipeline_events_contact": { + "name": "idx_pipeline_events_contact", + "columns": [ + { + "expression": "contact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_pipeline_events_match": { + "name": "idx_pipeline_events_match", + "columns": [ + { + "expression": "match_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_pipeline_events_type_occurred": { + "name": "idx_pipeline_events_type_occurred", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pipeline_events_org_id_orgs_id_fk": { + "name": "pipeline_events_org_id_orgs_id_fk", + "tableFrom": "pipeline_events", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "pipeline_events_contact_id_contacts_id_fk": { + "name": "pipeline_events_contact_id_contacts_id_fk", + "tableFrom": "pipeline_events", + "tableTo": "contacts", + "columnsFrom": [ + "contact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "pipeline_events_match_id_matches_id_fk": { + "name": "pipeline_events_match_id_matches_id_fk", + "tableFrom": "pipeline_events", + "tableTo": "matches", + "columnsFrom": [ + "match_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.application_effort_estimate": { + "name": "application_effort_estimate", + "schema": "public", + "values": [ + "loi_only", + "short_form", + "full_federal", + "unknown" + ] + }, + "public.contact_priority": { + "name": "contact_priority", + "schema": "public", + "values": [ + "named", + "generic" + ] + }, + "public.contact_source_provider": { + "name": "contact_source_provider", + "schema": "public", + "values": [ + "apollo", + "irs_990", + "website", + "manual" + ] + }, + "public.email_status": { + "name": "email_status", + "schema": "public", + "values": [ + "unverified", + "valid", + "risky", + "invalid" + ] + }, + "public.grant_source": { + "name": "grant_source", + "schema": "public", + "values": [ + "grants_gov", + "nh_state", + "irs_990pf", + "pnd_rss", + "candid", + "manual" + ] + }, + "public.grant_status": { + "name": "grant_status", + "schema": "public", + "values": [ + "open", + "expired", + "closed" + ] + }, + "public.icp_band": { + "name": "icp_band", + "schema": "public", + "values": [ + "below", + "primary", + "above", + "unknown" + ] + }, + "public.match_reject_reason": { + "name": "match_reject_reason", + "schema": "public", + "values": [ + "wrong_eligibility", + "wrong_geography", + "bad_capacity_fit", + "weak_mission_fit", + "stale_deadline", + "bad_contact", + "other" + ] + }, + "public.match_review_status": { + "name": "match_review_status", + "schema": "public", + "values": [ + "pending", + "approved", + "rejected", + "edited" + ] + }, + "public.pipeline_event_type": { + "name": "pipeline_event_type", + "schema": "public", + "values": [ + "enrolled", + "sent", + "opened", + "replied", + "bounced", + "unsubscribed", + "brief_requested", + "brief_sent", + "demo_booked", + "demo_held", + "pilot_started", + "converted" + ] + }, + "public.registration_status": { + "name": "registration_status", + "schema": "public", + "values": [ + "good_standing", + "lapsed", + "suspended", + "unknown" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/outreach-core/drizzle/server/meta/_journal.json b/packages/outreach-core/drizzle/server/meta/_journal.json index eb29057..5b11aab 100644 --- a/packages/outreach-core/drizzle/server/meta/_journal.json +++ b/packages/outreach-core/drizzle/server/meta/_journal.json @@ -22,6 +22,13 @@ "when": 1784235098035, "tag": "1784235098_match-uniques", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1784236406321, + "tag": "1784236406_funder-precedent-index", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/outreach-core/src/db/schema.ts b/packages/outreach-core/src/db/schema.ts index b3b4555..3d802f0 100644 --- a/packages/outreach-core/src/db/schema.ts +++ b/packages/outreach-core/src/db/schema.ts @@ -1,5 +1,6 @@ import { sql } from 'drizzle-orm'; import { + bigint, boolean, index, integer, @@ -139,6 +140,9 @@ export const grants = pgTable( .default(false), sourceUrl: text('source_url').notNull(), source: grantSourceEnum('source').notNull(), + // Links 990-PF-synthesized grants to their foundation for the + // precedent subscore; null for public-RFP sources. + funderEin: text('funder_ein'), status: grantStatusEnum('status').notNull().default('open'), synopsisEmbedding: vector('synopsis_embedding', { dimensions: 1536 }), lastVerifiedAt: timestamp('last_verified_at', { withTimezone: true }), @@ -151,6 +155,7 @@ export const grants = pgTable( index('idx_grants_status').on(t.status), index('idx_grants_close_date').on(t.closeDate), index('idx_grants_source').on(t.source), + index('idx_grants_funder_ein').on(t.funderEin), index('grants_synopsis_embedding_idx').using( 'hnsw', t.synopsisEmbedding.op('vector_cosine_ops'), @@ -339,6 +344,61 @@ export const pipelineEvents = pgTable( ], ); +/** + * Private foundations (990-PF filers) — the funder-precedent index's + * subjects. Discovered from the IRS BMF state files; grants-paid history + * parsed from their e-filed 990-PF XML. + */ +export const funders = pgTable( + 'funders', + { + id: uuid('id') + .primaryKey() + .default(sql`gen_random_uuid()`), + ein: text('ein').notNull(), + name: text('name').notNull(), + city: text('city'), + state: text('state'), + nteeCode: text('ntee_code'), + totalAssets: bigint('total_assets', { mode: 'number' }), + /** Part XV application info from the latest parsed filing (form/deadline/address text). */ + applicationInfo: jsonb('application_info'), + latestTaxYear: integer('latest_tax_year'), + latestObjectId: text('latest_object_id'), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow(), + }, + (t) => [ + uniqueIndex('idx_funders_ein').on(t.ein), + index('idx_funders_state').on(t.state), + ], +); + +/** One row per grant a foundation reported paying (990-PF Part XV line 3a). */ +export const funderGrants = pgTable( + 'funder_grants', + { + id: uuid('id') + .primaryKey() + .default(sql`gen_random_uuid()`), + funderId: uuid('funder_id') + .notNull() + .references(() => funders.id, { onDelete: 'cascade' }), + recipientName: text('recipient_name').notNull(), + recipientCity: text('recipient_city'), + recipientState: text('recipient_state'), + amount: integer('amount'), + purpose: text('purpose'), + taxYear: integer('tax_year').notNull(), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(), + }, + (t) => [ + index('idx_funder_grants_funder').on(t.funderId), + index('idx_funder_grants_funder_year').on(t.funderId, t.taxYear), + index('idx_funder_grants_recipient_state').on(t.recipientState), + ], +); + // --------------------------------------------------------------------------- // Schema barrel // --------------------------------------------------------------------------- @@ -350,4 +410,6 @@ export const schema = { contacts, matches, pipelineEvents, + funders, + funderGrants, }; diff --git a/packages/outreach-core/src/funders/actions/index.server.ts b/packages/outreach-core/src/funders/actions/index.server.ts new file mode 100644 index 0000000..040c48a --- /dev/null +++ b/packages/outreach-core/src/funders/actions/index.server.ts @@ -0,0 +1,2 @@ +export * from './upsert-funder.server.js'; +export * from './replace-funder-grants.server.js'; diff --git a/packages/outreach-core/src/funders/actions/replace-funder-grants.server.ts b/packages/outreach-core/src/funders/actions/replace-funder-grants.server.ts new file mode 100644 index 0000000..5de3633 --- /dev/null +++ b/packages/outreach-core/src/funders/actions/replace-funder-grants.server.ts @@ -0,0 +1,48 @@ +import { and, eq } from 'drizzle-orm'; + +import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; +import { schema } from '#~/db/db.js'; + +export interface FunderGrantInput { + readonly recipientName: string; + readonly recipientCity: string | null; + readonly recipientState: string | null; + readonly amount: number | null; + readonly purpose: string | null; +} + +/** + * Replaces a funder's grants-paid rows for one tax year — re-parsing the + * same filing is idempotent (990-PF rows have no stable per-grant id, so + * per-year replace beats per-row upsert). + */ +export async function serverReplaceFunderGrantsForYear( + db: NpOutreachDatabase | NpOutreachTransaction, + funderId: string, + taxYear: number, + grants: ReadonlyArray, +): Promise { + await db + .delete(schema.funderGrants) + .where( + and( + eq(schema.funderGrants.funderId, funderId), + eq(schema.funderGrants.taxYear, taxYear), + ), + ); + + for (let i = 0; i < grants.length; i += 500) { + const chunk = grants.slice(i, i + 500); + await db.insert(schema.funderGrants).values( + chunk.map((grant) => ({ + funderId, + taxYear, + recipientName: grant.recipientName, + recipientCity: grant.recipientCity, + recipientState: grant.recipientState, + amount: grant.amount, + purpose: grant.purpose, + })), + ); + } +} diff --git a/packages/outreach-core/src/funders/actions/upsert-funder.server.ts b/packages/outreach-core/src/funders/actions/upsert-funder.server.ts new file mode 100644 index 0000000..638c721 --- /dev/null +++ b/packages/outreach-core/src/funders/actions/upsert-funder.server.ts @@ -0,0 +1,42 @@ +import { sql } from 'drizzle-orm'; + +import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; +import { schema } from '#~/db/db.js'; + +export type NewFunderInput = Omit< + typeof schema.funders.$inferInsert, + 'id' | 'createdAt' | 'updatedAt' +>; + +/** Upserts a private foundation, keyed on EIN (BMF re-scans refresh in place). */ +export async function serverUpsertFunder( + db: NpOutreachDatabase | NpOutreachTransaction, + funder: NewFunderInput, +): Promise { + const [row] = await db + .insert(schema.funders) + .values(funder) + .onConflictDoUpdate({ + target: schema.funders.ein, + set: { + name: sql`excluded.name`, + city: sql`excluded.city`, + state: sql`excluded.state`, + nteeCode: sql`excluded.ntee_code`, + totalAssets: sql`excluded.total_assets`, + // Filing-derived fields only advance when the incoming parse is + // newer (or first): re-running discovery with null filing fields + // must not wipe an earlier XML parse. + applicationInfo: sql`COALESCE(excluded.application_info, funders.application_info)`, + latestTaxYear: sql`GREATEST(COALESCE(excluded.latest_tax_year, 0), COALESCE(funders.latest_tax_year, 0))`, + latestObjectId: sql`CASE WHEN COALESCE(excluded.latest_tax_year, 0) >= COALESCE(funders.latest_tax_year, 0) AND excluded.latest_object_id IS NOT NULL THEN excluded.latest_object_id ELSE funders.latest_object_id END`, + updatedAt: sql`now()`, + }, + }) + .returning({ id: schema.funders.id }); + + if (row == null) { + throw new Error('serverUpsertFunder: upsert returned no row'); + } + return row.id; +} diff --git a/packages/outreach-core/src/funders/index.server.ts b/packages/outreach-core/src/funders/index.server.ts new file mode 100644 index 0000000..5ac0a75 --- /dev/null +++ b/packages/outreach-core/src/funders/index.server.ts @@ -0,0 +1,2 @@ +export * from './actions/index.server.js'; +export * from './queries/index.server.js'; diff --git a/packages/outreach-core/src/funders/queries/index.server.ts b/packages/outreach-core/src/funders/queries/index.server.ts new file mode 100644 index 0000000..428fabe --- /dev/null +++ b/packages/outreach-core/src/funders/queries/index.server.ts @@ -0,0 +1,2 @@ +export * from './list-funder-synthesis-data.server.js'; +export * from './list-funder-latest-object-ids.server.js'; diff --git a/packages/outreach-core/src/funders/queries/list-funder-latest-object-ids.server.ts b/packages/outreach-core/src/funders/queries/list-funder-latest-object-ids.server.ts new file mode 100644 index 0000000..4a057c7 --- /dev/null +++ b/packages/outreach-core/src/funders/queries/list-funder-latest-object-ids.server.ts @@ -0,0 +1,34 @@ +import { inArray } from 'drizzle-orm'; + +import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; +import { schema } from '#~/db/db.js'; + +export interface FunderLatestObjectIdRow { + readonly ein: string; + readonly latestObjectId: string | null; +} + +/** + * Looks up the `latestObjectId` already recorded for a set of funder EINs — + * used by the 990-PF ingestion workflow to skip re-downloading/re-parsing a + * batch ZIP entry whose filing it has already parsed (the e-file index's + * selected filing for an EIN this run has the same object id as last run). + * Returns only rows that already exist as funders; EINs with no funder row + * yet are simply absent from the result (never re-processed unnecessarily). + */ +export async function serverListFunderLatestObjectIds( + db: NpOutreachDatabase | NpOutreachTransaction, + eins: ReadonlyArray, +): Promise { + if (eins.length === 0) return []; + + const rows = await db + .select({ + ein: schema.funders.ein, + latestObjectId: schema.funders.latestObjectId, + }) + .from(schema.funders) + .where(inArray(schema.funders.ein, [...eins])); + + return rows; +} diff --git a/packages/outreach-core/src/funders/queries/list-funder-synthesis-data.server.ts b/packages/outreach-core/src/funders/queries/list-funder-synthesis-data.server.ts new file mode 100644 index 0000000..aecb950 --- /dev/null +++ b/packages/outreach-core/src/funders/queries/list-funder-synthesis-data.server.ts @@ -0,0 +1,83 @@ +import { sql } from 'drizzle-orm'; + +import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; + +export interface FunderSynthesisRow { + funderId: string; + ein: string; + name: string; + city: string | null; + state: string | null; + applicationInfo: unknown; + latestTaxYear: number | null; + stateGrantCount: number; + medianAmount: number | null; + maxAmount: number | null; + /** Up to 12 distinct purpose strings from in-state grants, longest first. */ + purposes: string[]; +} + +/** + * Funders with enough in-state giving history to synthesize a grant row + * from ("this foundation funds orgs like you") — the 990-PF play for + * foundations with no public RFP. Aggregated per funder over grants paid + * into `recipientState`. + */ +export async function serverListFunderSynthesisData( + db: NpOutreachDatabase | NpOutreachTransaction, + { + recipientState, + minStateGrants, + }: { recipientState: string; minStateGrants: number }, +): Promise { + const result = await db.execute(sql` + SELECT + f.id AS funder_id, + f.ein, + f.name, + f.city, + f.state, + f.application_info, + f.latest_tax_year, + count(fg.id)::int AS state_grant_count, + percentile_cont(0.5) WITHIN GROUP (ORDER BY fg.amount) + FILTER (WHERE fg.amount IS NOT NULL) AS median_amount, + max(fg.amount) AS max_amount, + ( + SELECT array_agg(p.purpose) + FROM ( + SELECT DISTINCT fg2.purpose + FROM funder_grants fg2 + WHERE fg2.funder_id = f.id + AND fg2.recipient_state = ${recipientState} + AND fg2.purpose IS NOT NULL + AND length(fg2.purpose) > 3 + ORDER BY fg2.purpose + LIMIT 12 + ) p + ) AS purposes + FROM funders f + JOIN funder_grants fg ON fg.funder_id = f.id + WHERE fg.recipient_state = ${recipientState} + GROUP BY f.id + HAVING count(fg.id) >= ${minStateGrants} + ORDER BY count(fg.id) DESC + `); + + const { rows } = result as unknown as { + rows: Record[]; + }; + return rows.map((r) => ({ + funderId: r.funder_id as string, + ein: r.ein as string, + name: r.name as string, + city: (r.city as string) ?? null, + state: (r.state as string) ?? null, + applicationInfo: r.application_info ?? null, + latestTaxYear: (r.latest_tax_year as number) ?? null, + stateGrantCount: r.state_grant_count as number, + medianAmount: r.median_amount == null ? null : Math.round(Number(r.median_amount)), + maxAmount: r.max_amount == null ? null : Number(r.max_amount), + purposes: (r.purposes as string[]) ?? [], + })); +} diff --git a/packages/outreach-core/src/grants/queries/list-eligible-grants-for-org.server.ts b/packages/outreach-core/src/grants/queries/list-eligible-grants-for-org.server.ts index e0171db..9258e16 100644 --- a/packages/outreach-core/src/grants/queries/list-eligible-grants-for-org.server.ts +++ b/packages/outreach-core/src/grants/queries/list-eligible-grants-for-org.server.ts @@ -17,10 +17,15 @@ export interface EligibleGrantWithSimilarity { | 'full_federal' | 'unknown'; applicationFormSupported: boolean; + funderEin: string | null; similarity: number; + /** Funder's historical grant count into the org's state (990-PF index); null when the grant has no linked funder. */ + funderStateGrantCount: number | null; } export interface EligibleGrantFilters { + /** Org's state, for the funder-precedent count. */ + readonly orgState: string; /** Days of runway the deadline must clear (hard gate: 21). */ readonly minDaysToDeadline: number; /** Minimum award ceiling in dollars (hard gate: 10_000). */ @@ -55,13 +60,20 @@ export async function serverListEligibleGrantsForOrg( awardCeiling: schema.grants.awardCeiling, applicationEffortEstimate: schema.grants.applicationEffortEstimate, applicationFormSupported: schema.grants.applicationFormSupported, + funderEin: schema.grants.funderEin, similarity: sql`1 - (${schema.grants.synopsisEmbedding} <=> ${vector}::vector)`, + funderStateGrantCount: sql`( + SELECT count(*)::int FROM funder_grants fg + JOIN funders f ON fg.funder_id = f.id + WHERE f.ein = ${schema.grants.funderEin} + AND fg.recipient_state = ${filters.orgState} + )`, }) .from(schema.grants) .where( sql`${schema.grants.status} = 'open' AND ${schema.grants.synopsisEmbedding} IS NOT NULL - AND ${schema.grants.closeDate} >= now() + make_interval(days => ${filters.minDaysToDeadline}) + AND (${schema.grants.closeDate} IS NULL OR ${schema.grants.closeDate} >= now() + make_interval(days => ${filters.minDaysToDeadline})) AND ${schema.grants.awardCeiling} >= ${filters.minAwardCeiling}`, ) .orderBy(sql`${schema.grants.synopsisEmbedding} <=> ${vector}::vector`) diff --git a/packages/outreach-core/src/index.server.ts b/packages/outreach-core/src/index.server.ts index 7196244..aa63d49 100644 --- a/packages/outreach-core/src/index.server.ts +++ b/packages/outreach-core/src/index.server.ts @@ -4,3 +4,4 @@ export * from './grants/index.server.js'; export * from './matches/index.server.js'; export * from './orgs/index.server.js'; export * from './pipeline/index.server.js'; +export * from './funders/index.server.js'; diff --git a/packages/outreach-core/src/matches/hard-gates.test.ts b/packages/outreach-core/src/matches/hard-gates.test.ts index 6c2d019..5b2edb6 100644 --- a/packages/outreach-core/src/matches/hard-gates.test.ts +++ b/packages/outreach-core/src/matches/hard-gates.test.ts @@ -90,14 +90,13 @@ describe('evaluateHardGates', () => { expect(result.failures).not.toContain('deadline_too_soon'); }); - it('fails when there is no close date at all', () => { + it('treats a missing close date as a rolling deadline (passes the gate)', () => { const result = evaluateHardGates( org, { ...grant, closeDate: null }, { now: NOW }, ); - expect(result.passed).toBe(false); - expect(result.failures).toContain('deadline_too_soon'); + expect(result.failures).not.toContain('deadline_too_soon'); }); it('fails when the award ceiling is below the minimum', () => { @@ -136,7 +135,7 @@ describe('evaluateHardGates', () => { { eligibilityEntityTypes: ['501c3'], geographicScope: 'California', - closeDate: null, + closeDate: daysFromNow(5), awardCeiling: null, applicationFormSupported: false, }, diff --git a/packages/outreach-core/src/matches/hard-gates.ts b/packages/outreach-core/src/matches/hard-gates.ts index ec2ab00..00318c3 100644 --- a/packages/outreach-core/src/matches/hard-gates.ts +++ b/packages/outreach-core/src/matches/hard-gates.ts @@ -194,7 +194,11 @@ function isGeographyEligible( } function hasSufficientRunway(grant: HardGateGrantInput, now: Date): boolean { - if (grant.closeDate == null) return false; + // Null close date = rolling/no stated deadline (typical for private + // foundations found via 990-PF). Rolling is pitchable — the runway + // SUBSCORE keeps it un-urgent; the GATE only kills real, too-soon + // deadlines. + if (grant.closeDate == null) return true; const msPerDay = 24 * 60 * 60 * 1000; const daysRemaining = (grant.closeDate.getTime() - now.getTime()) / msPerDay; return daysRemaining >= MIN_DAYS_TO_DEADLINE; diff --git a/packages/outreach-core/src/matches/scoring.test.ts b/packages/outreach-core/src/matches/scoring.test.ts index 6d67b5c..dad46aa 100644 --- a/packages/outreach-core/src/matches/scoring.test.ts +++ b/packages/outreach-core/src/matches/scoring.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { capacityFitSubscore, + funderPrecedentSubscore, competitionSubscore, EASY_WIN_THRESHOLD, effortSubscore, @@ -79,7 +80,34 @@ describe('runwaySubscore', () => { }); }); +describe('funderPrecedentSubscore', () => { + it('tiers repeated in-state giving, zero without evidence', () => { + expect(funderPrecedentSubscore(null)).toBe(0); + expect(funderPrecedentSubscore(0)).toBe(0); + expect(funderPrecedentSubscore(1)).toBe(8); + expect(funderPrecedentSubscore(3)).toBe(15); + expect(funderPrecedentSubscore(5)).toBe(20); + expect(funderPrecedentSubscore(10)).toBe(25); + }); +}); + describe('scoreMatch', () => { + it('withholds easy-win from precedent-less high scorers', () => { + const result = scoreMatch({ + similarity: 0.75, + orgTotalRevenue: 1_000_000, + awardCeiling: 200_000, + geographicScope: 'New Hampshire', + applicationEffortEstimate: 'loi_only', + closeDate: weeksFromNow(6), + now: NOW, + funderStateGrantCount: null, + }); + // 30+15+15+10+5 = 75 — over the threshold but no precedent floor. + expect(result.totalScore).toBe(75); + expect(result.easyWin).toBe(false); + }); + it('sums subscores and flags easy wins', () => { const result = scoreMatch({ similarity: 0.75, @@ -89,11 +117,12 @@ describe('scoreMatch', () => { applicationEffortEstimate: 'short_form', closeDate: weeksFromNow(6), now: NOW, + funderStateGrantCount: 6, }); - // 30 fit + 15 capacity + 15 competition + 8 effort + 5 runway - expect(result.totalScore).toBe(73); + // 30 fit + 20 precedent + 15 capacity + 15 competition + 8 effort + 5 runway + expect(result.totalScore).toBe(93); expect(result.easyWin).toBe(true); - expect(result.subscores.funderPrecedent).toBe(0); + expect(result.subscores.funderPrecedent).toBe(20); }); it('keeps weak matches under the easy-win line', () => { @@ -105,6 +134,7 @@ describe('scoreMatch', () => { applicationEffortEstimate: 'full_federal', closeDate: weeksFromNow(2), now: NOW, + funderStateGrantCount: null, }); expect(result.totalScore).toBeLessThan(EASY_WIN_THRESHOLD); expect(result.easyWin).toBe(false); diff --git a/packages/outreach-core/src/matches/scoring.ts b/packages/outreach-core/src/matches/scoring.ts index 731987d..d4d359e 100644 --- a/packages/outreach-core/src/matches/scoring.ts +++ b/packages/outreach-core/src/matches/scoring.ts @@ -4,16 +4,15 @@ * the match workflow supplies the embedding similarity, everything else * derives from columns. * - * v1 weights (funder precedent's 25 points are NOT yet awarded — the - * 990-PF index is a later deliverable, so the achievable maximum is 75, - * not 100). `subscores` records each component so weights can be re-tuned - * from review/booking data without re-deriving inputs. + * `subscores` records each component so weights can be re-tuned from + * review/booking data without re-deriving inputs. * - * mission fit 30 embedding cosine similarity, scaled - * capacity fit 15 award ceiling vs org revenue (sweet spot 10–75%) - * competition 15 state/NH-restricted pools beat national ones - * effort 10 LOI/short-form beat full federal - * runway 5 3–10 weeks to deadline is ideal + * mission fit 30 embedding cosine similarity, scaled + * funder precedent 25 historical giving into the org's state (990-PF) + * capacity fit 15 award ceiling vs org revenue (sweet spot 10–75%) + * competition 15 state/NH-restricted pools beat national ones + * effort 10 LOI/short-form beat full federal + * runway 5 3–10 weeks to deadline is ideal */ export interface MatchSubscores { @@ -22,8 +21,7 @@ export interface MatchSubscores { readonly competition: number; readonly effort: number; readonly runway: number; - /** Not yet computed — reserved so the jsonb shape is stable. */ - readonly funderPrecedent: 0; + readonly funderPrecedent: number; } export interface ScoreMatchInput { @@ -39,16 +37,25 @@ export interface ScoreMatchInput { | 'unknown'; readonly closeDate: Date | null; readonly now: Date; + /** + * Historical grants this funder has paid to recipients in the org's + * state (from the 990-PF index). Null = no precedent data for this + * grant's funder (e.g. federal agencies) — scores 0, not neutral: the + * plan weights precedent as the strongest single predictor, and absence + * of evidence should rank below presence. + */ + readonly funderStateGrantCount: number | null; } -export const ACHIEVABLE_MAX_SCORE = 75; +export const ACHIEVABLE_MAX_SCORE = 100; /** - * "Easy win" threshold, v1: two-thirds of the achievable maximum. The - * plan's full definition also requires a funder-precedent floor — that - * gate returns when the 990-PF index lands; thresholds re-tune on review - * and demo-booking data regardless. + * "Easy win" threshold. With the 990-PF precedent subscore live the scale + * is the plan's full 0–100; the plan's >=75 easy-win bar applies, plus its + * precedent floor (see scoreMatch). Thresholds re-tune on review and + * demo-booking data. */ -export const EASY_WIN_THRESHOLD = 50; +export const EASY_WIN_THRESHOLD = 65; +export const EASY_WIN_MIN_PRECEDENT = 12; /** Similarity below this scores 0 fit; above the ceiling scores full fit. */ const SIMILARITY_FLOOR = 0.45; @@ -115,6 +122,22 @@ export function effortSubscore( } } +/** + * Funder precedent (25): "a foundation that gave to three NH orgs like + * this one is a near-certain match for a fourth" — the plan's strongest + * single predictor. v1 measures repeated giving into the org's state; + * NTEE-level matching arrives when recipient orgs get resolved to EINs. + */ +export function funderPrecedentSubscore( + funderStateGrantCount: number | null, +): number { + if (funderStateGrantCount == null || funderStateGrantCount <= 0) return 0; + if (funderStateGrantCount >= 10) return 25; + if (funderStateGrantCount >= 5) return 20; + if (funderStateGrantCount >= 3) return 15; + return 8; +} + const MS_PER_WEEK = 7 * 24 * 60 * 60 * 1000; /** 3–10 weeks out is ideal: urgent enough to act on, long enough to apply. */ @@ -140,7 +163,7 @@ export function scoreMatch(input: ScoreMatchInput): ScoredMatch { competition: competitionSubscore(input.geographicScope), effort: effortSubscore(input.applicationEffortEstimate), runway: runwaySubscore(input.closeDate, input.now), - funderPrecedent: 0, + funderPrecedent: funderPrecedentSubscore(input.funderStateGrantCount), }; const totalScore = @@ -148,11 +171,14 @@ export function scoreMatch(input: ScoreMatchInput): ScoredMatch { subscores.capacityFit + subscores.competition + subscores.effort + - subscores.runway; + subscores.runway + + subscores.funderPrecedent; return { totalScore, subscores, - easyWin: totalScore >= EASY_WIN_THRESHOLD, + easyWin: + totalScore >= EASY_WIN_THRESHOLD && + subscores.funderPrecedent >= EASY_WIN_MIN_PRECEDENT, }; } diff --git a/packages/outreach-core/src/orgs/queries/list-match-candidate-orgs.server.ts b/packages/outreach-core/src/orgs/queries/list-match-candidate-orgs.server.ts index 3cb7390..7bfc41f 100644 --- a/packages/outreach-core/src/orgs/queries/list-match-candidate-orgs.server.ts +++ b/packages/outreach-core/src/orgs/queries/list-match-candidate-orgs.server.ts @@ -1,4 +1,4 @@ -import { and, eq } from 'drizzle-orm'; +import { and, eq, or, isNull, sql } from 'drizzle-orm'; import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; import { schema } from '#~/db/db.js'; @@ -8,14 +8,19 @@ export interface MatchCandidateOrg { name: string; city: string | null; state: string; + ein: string | null; nteeCode: string | null; totalRevenue: number | null; } /** * Orgs eligible to enter match scoring: the primary ICP band ($100K–$5M - * revenue), NH-based, in good standing with the registry. Everything else - * is either not the customer (yet) or would fail the standing gate anyway. + * revenue), NH-based, in good standing with the registry — and not + * themselves grantmakers (NTEE major group T): foundations and charitable + * trusts register with NHDOJ like any charity and often land in the ICP + * revenue band, but they GIVE grants, they don't seek them (surfaced by + * the first precedent-scored run, where the top "leads" were foundations + * matched to themselves). */ export async function serverListMatchCandidateOrgs( db: NpOutreachDatabase | NpOutreachTransaction, @@ -27,6 +32,7 @@ export async function serverListMatchCandidateOrgs( name: schema.orgs.name, city: schema.orgs.city, state: schema.orgs.state, + ein: schema.orgs.ein, nteeCode: schema.orgs.nteeCode, totalRevenue: schema.orgs.totalRevenue, }) @@ -36,6 +42,10 @@ export async function serverListMatchCandidateOrgs( eq(schema.orgs.state, 'NH'), eq(schema.orgs.registrationStatus, 'good_standing'), eq(schema.orgs.icpBand, 'primary'), + or( + isNull(schema.orgs.nteeCode), + sql`${schema.orgs.nteeCode} NOT LIKE 'T%'`, + ), ), ) .limit(limit); diff --git a/yarn.lock b/yarn.lock index 3d426bf..4232d14 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1612,6 +1612,7 @@ __metadata: drizzle-orm: "npm:0.44.6" esbuild: "npm:^0.24.0" fast-xml-parser: "npm:^4.5.0" + fflate: "npm:^0.8.2" pdfjs-dist: "npm:^4.10.38" pg: "npm:8.20.0" tsx: "npm:^4.19.2" @@ -3749,6 +3750,13 @@ __metadata: languageName: node linkType: hard +"fflate@npm:^0.8.2": + version: 0.8.3 + resolution: "fflate@npm:0.8.3" + checksum: 10c0/eab181ca37f5348ae76d4b6f840e0026e30220e33153289ac942222d8b9638237d486507dbcc09878d724095bd354993a2ee48bbee99c8f2c6440d4448719aa7 + languageName: node + linkType: hard + "fill-range@npm:^7.1.1": version: 7.1.1 resolution: "fill-range@npm:7.1.1"