christian/grant-outreach-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ingestion): survive first contact with real data sources

Browse Source 
NHDOJ: parser rebuilt for the real 8-column registry layout (Reg. No. |
Charity Name | Address | City | State | Zip | Status | Report Due) with
single-letter G/X/S statuses; Reg. No. is the stable upsert key (new
orgs.registration_number column + partial unique index, enum gains
'suspended' via idempotent ADD VALUE); out-of-state registrants keep
their real state. Akamai-safe fetch headers + NHDOJ_REGISTRY_PDF_PATH
local-file override.

ProPublica: zero-hit state-scoped searches return 404, not an empty
list — map to no-candidates instead of failure (tripped the systemic-
failure breaker at 60/200 on first contact). Enrichment queue now
prioritizes NH good-standing orgs over the out-of-state tail.

PND: feed retired upstream (HTML shell on every historical path) —
documented as rework candidate, low priority.

run-once.ts: supervised one-off runner through the durable DBOS
handles (workflow modules now export run*Now accessors); drop the
double pool.end() after DBOS.shutdown().

First supervised run: 200 Grants.gov opportunities (1 auto-expired),
13,632 orgs from the 427-page registry, enrichment at failed=0 with
121/200 EIN resolution in the NH-priority batch.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
2026-07-16 16:20:47 -04:00
parent fe12b032af
commit 8d57d57557
17 changed files with 1898 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
apps/outreach-worker/.env.example
View File

@@ -16,6 +16,9 @@ GCP_SERVICE_ACCOUNT_KEY_PATH=./secrets/gcp-service-account.json
# NHDOJ Charitable Trusts registry PDF — the URL changes whenever NHDOJ # NHDOJ Charitable Trusts registry PDF — the URL changes whenever NHDOJ
# republishes (roughly monthly). When unset, the monthly ingestNhdojOrgs # republishes (roughly monthly). When unset, the monthly ingestNhdojOrgs
# workflow logs a warning and no-ops instead of failing. # workflow logs a warning and no-ops instead of failing.
# Local-file override for supervised runs / Akamai outages (takes
# precedence over the URL):
# NHDOJ_REGISTRY_PDF_PATH=./secrets/registered-charities.pdf
# NHDOJ_REGISTRY_PDF_URL=https://www.doj.nh.gov/.../charitable-trusts-registry.pdf # NHDOJ_REGISTRY_PDF_URL=https://www.doj.nh.gov/.../charitable-trusts-registry.pdf
# Optional override for the Philanthropy News Digest RFP feed (defaults to # Optional override for the Philanthropy News Digest RFP feed (defaults to

120
apps/outreach-worker/src/run-once.ts Normal file
View File

@@ -0,0 +1,120 @@
/**
* Supervised one-off workflow runner.
*
* DATABASE_URL=... node --import tsx/esm src/run-once.ts <workflow> [...]
*
* where <workflow> is one or more of: ingestGrants, ingestPndRss,
* ingestNhdojOrgs, enrichOrgs, expireGrants — or `all` for the standard
* first-run order (grants → pnd → nhdoj → expire → enrich).
*
* Boots exactly like main.ts (same deps injection, same DBOS launch — see
* that file's boot-order comment), runs the requested workflows through
* their durable DBOS handles sequentially, then shuts down. Scheduled crons
* ARE active while this process lives; runs are short enough that this
* doesn't matter in practice.
*/
import { DBOS } from '@dbos-inc/dbos-sdk';
import { DrizzleDataSource } from '@dbos-inc/drizzle-datasource';
import { schema } from '@novelpad/outreach-core';
import { drizzle } from 'drizzle-orm/node-postgres';
import pg from 'pg';
import { runEnrichOrgsNow, setEnrichOrgsDeps } from './workflows/enrich-orgs.js';
import {
runExpireGrantsNow,
setExpireGrantsDeps,
} from './workflows/expire-grants.js';
import {
runIngestGrantsNow,
setIngestGrantsDeps,
} from './workflows/ingest-grants.js';
import {
runIngestNhdojOrgsNow,
setIngestNhdojOrgsDeps,
} from './workflows/ingest-nhdoj-orgs.js';
import {
runIngestPndRssNow,
setIngestPndRssDeps,
} from './workflows/ingest-pnd-rss.js';
const RUNNERS: Record<string, () => Promise<void>> = {
ingestGrants: runIngestGrantsNow,
ingestPndRss: runIngestPndRssNow,
ingestNhdojOrgs: runIngestNhdojOrgsNow,
enrichOrgs: runEnrichOrgsNow,
expireGrants: runExpireGrantsNow,
};
const FIRST_RUN_ORDER = [
'ingestGrants',
'ingestPndRss',
'ingestNhdojOrgs',
'expireGrants',
'enrichOrgs',
];
if (process.env.DATABASE_URL == null) {
throw new Error('run-once: DATABASE_URL is required');
}
const requested = process.argv.slice(2);
const names = requested.includes('all') ? FIRST_RUN_ORDER : requested;
if (names.length === 0 || names.some((n) => RUNNERS[n] == null)) {
console.error(
`Usage: run-once.ts <workflow...>\nKnown workflows: ${Object.keys(RUNNERS).join(', ')}, all`,
);
process.exit(2);
}
const { Pool } = pg;
const dbosClientConfig = {
connectionString: process.env.DATABASE_URL,
application_name: 'helmdocs-outreach-run-once',
} satisfies pg.ClientConfig;
const pool = new Pool({
...dbosClientConfig,
max: Number(process.env.PG_POOL_MAX ?? 20),
});
const db = drizzle(pool, { schema });
async function main() {
await DrizzleDataSource.initializeDBOSSchema(dbosClientConfig);
setIngestGrantsDeps({ db });
setExpireGrantsDeps({ db });
setIngestPndRssDeps({ db });
setIngestNhdojOrgsDeps({ db });
setEnrichOrgsDeps({ db });
DBOS.setConfig({
name: 'helmdocs-outreach-worker',
systemDatabasePool: pool,
runAdminServer: false,
});
await DBOS.launch();
let failed = false;
for (const name of names) {
console.log(`\n=== run-once: ${name} ===`);
const startedAt = Date.now();
try {
await RUNNERS[name]!();
console.log(
`=== run-once: ${name} OK in ${((Date.now() - startedAt) / 1000).toFixed(1)}s ===`,
);
} catch (err) {
failed = true;
console.error(`=== run-once: ${name} FAILED ===`, err);
}
}
// DBOS.shutdown() ends the pool we handed it via systemDatabasePool —
// a second pool.end() here throws "Called end on pool more than once".
await DBOS.shutdown();
process.exit(failed ? 1 : 0);
}
main().catch((err) => {
console.error('[run-once] fatal:', err);
process.exit(1);
});

421
apps/outreach-worker/src/sources/nhdoj/parse-registry.test.ts
View File

@@ -5,236 +5,303 @@ import {
normalizeRegistrationStatus, normalizeRegistrationStatus,
normalizeRegistryRows, normalizeRegistryRows,
reconstructRegistryRows, reconstructRegistryRows,
type RegistryRow,
} from './parse-registry.js'; } from './parse-registry.js';
/** Column x-starts used across fixtures: name @50, city @300, status @450. */ // x-anchors observed in the real PDF (2026-07-08 republish).
const NAME_X = 50; const X = {
const CITY_X = 300; regNo: 17,
const STATUS_X = 450; name: 70,
address: 380,
city: 627,
state: 710,
zip: 748,
status: 820,
reportDue: 891,
} as const;
function item(str: string, x: number, y: number): PositionedTextItem { function item(str: string, x: number, y: number): PositionedTextItem {
return { str, x, y }; return { str, x, y };
} }
function headerRow(y: number): PositionedTextItem[] { function headerLine(y: number): PositionedTextItem[] {
return [ return [
item('Organization', NAME_X, y), item('Reg. No.', X.regNo, y),
item('City', CITY_X, y), item('Charity Name', X.name, y),
item('Status', STATUS_X, y), item('Address', X.address, y),
item('City', X.city, y),
item('State', X.state, y),
item('Zip', X.zip, y),
item('Status', X.status, y),
item('Report Due', X.reportDue, y),
];
}
function dataLine(
y: number,
cells: {
regNo?: string;
name?: string;
address?: string;
city?: string;
state?: string;
zip?: string;
status?: string;
reportDue?: string;
},
): PositionedTextItem[] {
const out: PositionedTextItem[] = [];
if (cells.regNo) out.push(item(cells.regNo, X.regNo, y));
if (cells.name) out.push(item(cells.name, X.name + 2, y));
if (cells.address) out.push(item(cells.address, X.address, y));
if (cells.city) out.push(item(cells.city, X.city, y));
if (cells.state) out.push(item(cells.state, X.state, y));
if (cells.zip) out.push(item(cells.zip, X.zip, y));
if (cells.status) out.push(item(cells.status, X.status, y));
if (cells.reportDue) out.push(item(cells.reportDue, X.reportDue, y));
return out;
}
function letterheadAndLegend(): PositionedTextItem[] {
return [
item('New Hampshire Department of Justice', 15, 580),
item('Registered Charities List', 450, 581),
item('Charitable Trusts Unit', 896, 580),
item(
'G = Good Standing; X = Not in Good Standing; S = Suspended',
365,
569,
),
]; ];
} }
describe('reconstructRegistryRows', () => { describe('reconstructRegistryRows', () => {
it('returns no rows for an empty page', () => { it('parses a full page: letterhead, legend, header, data rows', () => {
expect(reconstructRegistryRows([[]])).toEqual([]);
expect(reconstructRegistryRows([])).toEqual([]);
});
it('infers column boundaries from the header row and buckets data by x', () => {
const page = [ const page = [
...headerRow(900), ...letterheadAndLegend(),
item('Acme Foundation', NAME_X, 880), ...headerLine(547),
item('Concord', CITY_X, 880), ...dataLine(532, {
item('Good Standing', STATUS_X, 880), regNo: '35309',
name: '22ZERO Follow Me In',
address: 'PO Box 23',
city: 'Pulaski',
state: 'TN',
zip: '38478',
status: 'G',
reportDue: '5/15/2027',
}),
...dataLine(518, {
regNo: '33454',
name: '#HappyPeriod',
address: '4911 Mountain View Drive',
city: 'Palmdale',
state: 'CA',
zip: '93552',
status: 'X',
reportDue: '9/14/2026',
}),
item('Updated: July 08, 2026', 15, 60),
]; ];
const rows = reconstructRegistryRows([page]); const rows = reconstructRegistryRows([page]);
expect(rows).toEqual([
{ name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' },
]);
});
it('infers boundaries correctly even when they differ from other fixtures', () => {
const page = [
item('Organization', 40, 900),
item('City', 250, 900),
item('Status', 500, 900),
// Slightly right of each boundary — still buckets into the same column.
item('Bright Futures NH', 42, 870),
item('Nashua', 255, 870),
item('Active', 505, 870),
];
const rows = reconstructRegistryRows([page]);
expect(rows).toEqual([
{ name: 'Bright Futures NH', city: 'Nashua', status: 'Active' },
]);
});
it('folds a multi-line org name (continuation line has empty city/status) into the previous row', () => {
const page = [
...headerRow(900),
item('Very Long Nonprofit', NAME_X, 860),
item('Manchester', CITY_X, 860),
item('Lapsed', STATUS_X, 860),
// Wrapped second line of the same org name — no city/status text.
item('Name Incorporated', NAME_X, 840),
];
const rows = reconstructRegistryRows([page]);
expect(rows).toEqual([ expect(rows).toEqual([
{ {
name: 'Very Long Nonprofit Name Incorporated', registrationNumber: '35309',
city: 'Manchester', name: '22ZERO Follow Me In',
status: 'Lapsed', address: 'PO Box 23',
city: 'Pulaski',
state: 'TN',
zip: '38478',
status: 'G',
reportDue: '5/15/2027',
},
{
registrationNumber: '33454',
name: '#HappyPeriod',
address: '4911 Mountain View Drive',
city: 'Palmdale',
state: 'CA',
zip: '93552',
status: 'X',
reportDue: '9/14/2026',
}, },
]); ]);
}); });
it('supports multi-word continuation lines split across several text items', () => { it('folds a wrapped name line into the previous row', () => {
const page = [ const page = [
...headerRow(900), ...headerLine(547),
item('Friends of the', NAME_X, 860), ...dataLine(532, {
item('Merrimack', CITY_X, 860), regNo: '30456',
item('Current', STATUS_X, 860), name: '1st New Hampshire Light Battery',
item('River', NAME_X, 840), address: '11 Pinecrest Circle',
item('Watershed', NAME_X + 40, 840), city: 'Bedford',
state: 'NH',
zip: '03110',
status: 'X',
}),
// Continuation: name column only — no reg no, no status.
...dataLine(518, { name: 'Historical Association' }),
]; ];
const rows = reconstructRegistryRows([page]); const rows = reconstructRegistryRows([page]);
expect(rows).toHaveLength(1);
expect(rows).toEqual([ expect(rows[0]!.name).toBe(
{ '1st New Hampshire Light Battery Historical Association',
name: 'Friends of the River Watershed', );
city: 'Merrimack',
status: 'Current',
},
]);
}); });
it('drops header, letterhead, and page-number footer lines', () => { it('folds wrapped address text without disturbing the name', () => {
const page = [ const page = [
item('State of New Hampshire Department of Justice', NAME_X, 950), ...headerLine(547),
...headerRow(900), ...dataLine(532, {
item('Acme Foundation', NAME_X, 880), regNo: '32030',
item('Concord', CITY_X, 880), name: '#WalkAway Foundation',
item('Good Standing', STATUS_X, 880), address: '10521 Judicial Drive, Suite 200-A',
item('3', NAME_X, 50), city: 'Fairfax',
state: 'VA',
zip: '22030',
status: 'G',
}),
...dataLine(518, { address: 'Fairfax, VA 22030' }),
]; ];
const rows = reconstructRegistryRows([page]); const rows = reconstructRegistryRows([page]);
expect(rows).toHaveLength(1);
expect(rows).toEqual([ expect(rows[0]!.name).toBe('#WalkAway Foundation');
{ name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' }, expect(rows[0]!.address).toBe(
]); '10521 Judicial Drive, Suite 200-A Fairfax, VA 22030',
);
}); });
it('drops a "Page X of Y" footer line', () => { it('reuses the last-known anchors on header-less continuation pages', () => {
const page = [
...headerRow(900),
item('Acme Foundation', NAME_X, 880),
item('Concord', CITY_X, 880),
item('Good Standing', STATUS_X, 880),
item('Page 1 of 12', NAME_X, 50),
];
const rows = reconstructRegistryRows([page]);
expect(rows).toEqual([
{ name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' },
]);
});
it('accepts explicit columnBoundaries for a continuation page with no repeated header', () => {
const page = [
item('Second Page Org', NAME_X, 900),
item('Keene', CITY_X, 900),
item('Unknown', STATUS_X, 900),
];
const rows = reconstructRegistryRows([page], {
columnBoundaries: [NAME_X, CITY_X, STATUS_X],
});
expect(rows).toEqual([
{ name: 'Second Page Org', city: 'Keene', status: 'Unknown' },
]);
});
it('reuses the last inferred boundaries across pages whose header does not repeat', () => {
const page1 = [ const page1 = [
...headerRow(900), ...headerLine(547),
item('First Page Org', NAME_X, 880), ...dataLine(532, {
item('Concord', CITY_X, 880), regNo: '1',
item('Good Standing', STATUS_X, 880), name: 'Alpha',
city: 'Concord',
state: 'NH',
status: 'G',
}),
]; ];
const page2 = [ const page2 = [
item('Second Page Org', NAME_X, 900), ...dataLine(532, {
item('Keene', CITY_X, 900), regNo: '2',
item('Lapsed', STATUS_X, 900), name: 'Beta',
city: 'Nashua',
state: 'NH',
status: 'S',
}),
]; ];
const rows = reconstructRegistryRows([page1, page2]); const rows = reconstructRegistryRows([page1, page2]);
expect(rows.map((r) => r.name)).toEqual(['Alpha', 'Beta']);
expect(rows).toEqual([
{ name: 'First Page Org', city: 'Concord', status: 'Good Standing' },
{ name: 'Second Page Org', city: 'Keene', status: 'Lapsed' },
]);
}); });
it('throws when a page has data but no boundaries can be determined', () => { it('throws loudly when no anchors can be determined', () => {
const page = [item('Mystery Org', NAME_X, 900)]; const page = [
...dataLine(532, { regNo: '1', name: 'Alpha', status: 'G' }),
];
expect(() => reconstructRegistryRows([page])).toThrow(/column anchors/);
});
expect(() => reconstructRegistryRows([page])).toThrow( it('accepts explicit anchors and skips empty pages', () => {
/could not determine column boundaries/, const anchors = [
); X.regNo,
X.name,
X.address,
X.city,
X.state,
X.zip,
X.status,
X.reportDue,
] as const;
const page = [
...dataLine(532, { regNo: '9', name: 'Gamma', status: 'G' }),
];
const rows = reconstructRegistryRows([[], page], {
columnAnchors: anchors,
});
expect(rows).toEqual([
{
registrationNumber: '9',
name: 'Gamma',
address: null,
city: null,
state: null,
zip: null,
status: 'G',
reportDue: null,
},
]);
}); });
}); });
describe('normalizeRegistrationStatus', () => { describe('normalizeRegistrationStatus', () => {
it.each([ it('maps the registry letter codes', () => {
['Good Standing', 'good_standing'], expect(normalizeRegistrationStatus('G')).toBe('good_standing');
['GOOD STANDING', 'good_standing'], expect(normalizeRegistrationStatus('g')).toBe('good_standing');
['Current', 'good_standing'], expect(normalizeRegistrationStatus('X')).toBe('lapsed');
['Active', 'good_standing'], expect(normalizeRegistrationStatus('S')).toBe('suspended');
['Lapsed', 'lapsed'], });
['Delinquent', 'lapsed'],
['Suspended', 'lapsed'], it('maps spelled-out fallbacks, including the negated phrase', () => {
['Expired', 'lapsed'], expect(normalizeRegistrationStatus('Good Standing')).toBe('good_standing');
['Revoked', 'lapsed'], expect(normalizeRegistrationStatus('Not in Good Standing')).toBe('lapsed');
['', 'unknown'], expect(normalizeRegistrationStatus('Suspended')).toBe('suspended');
['Pending Review', 'unknown'], expect(normalizeRegistrationStatus('Revoked')).toBe('lapsed');
] as const)('maps %s -> %s', (input, expected) => { });
expect(normalizeRegistrationStatus(input)).toBe(expected);
it('maps blanks and surprises to unknown', () => {
expect(normalizeRegistrationStatus('')).toBe('unknown');
expect(normalizeRegistrationStatus(' ')).toBe('unknown');
expect(normalizeRegistrationStatus('Q')).toBe('unknown');
}); });
}); });
describe('normalizeRegistryRows', () => { describe('normalizeRegistryRows', () => {
it('maps status variants, trims/collapses whitespace, and preserves a null city', () => { it('trims to org fields, uppercases state, drops artifacts', () => {
const rows: RegistryRow[] = [ const rows = normalizeRegistryRows([
{ name: ' Acme Foundation ', city: ' Concord ', status: 'Good Standing' }, {
{ name: 'Delinquent Org', city: null, status: 'Delinquent' }, registrationNumber: '35309',
{ name: 'Mystery Org', city: 'Keene', status: 'Something Else' }, name: ' 22ZERO Follow Me In ',
]; address: 'PO Box 23',
city: ' Pulaski ',
expect(normalizeRegistryRows(rows)).toEqual([ state: 'tn',
{ name: 'Acme Foundation', city: 'Concord', status: 'good_standing' }, zip: '38478',
{ name: 'Delinquent Org', city: null, status: 'lapsed' }, status: 'G',
{ name: 'Mystery Org', city: 'Keene', status: 'unknown' }, reportDue: '5/15/2027',
},
{
registrationNumber: null,
name: 'Updated: July 08, 2026',
address: null,
city: null,
state: null,
zip: null,
status: '',
reportDue: null,
},
{
registrationNumber: null,
name: '',
address: null,
city: null,
state: null,
zip: null,
status: 'G',
reportDue: null,
},
]); ]);
});
it('drops rows with an empty name and header/footer artifact rows that slipped through', () => { expect(rows).toEqual([
const rows: RegistryRow[] = [ {
{ name: '', city: 'Concord', status: 'Good Standing' }, registrationNumber: '35309',
{ name: 'Organization City Status', city: null, status: '' }, name: '22ZERO Follow Me In',
{ name: 'Real Org', city: 'Nashua', status: 'Active' }, city: 'Pulaski',
]; state: 'TN',
status: 'good_standing',
expect(normalizeRegistryRows(rows)).toEqual([ },
{ name: 'Real Org', city: 'Nashua', status: 'good_standing' },
]);
});
it('collapses an empty-string city to null', () => {
const rows: RegistryRow[] = [{ name: 'Org', city: ' ', status: 'Active' }];
expect(normalizeRegistryRows(rows)).toEqual([
{ name: 'Org', city: null, status: 'good_standing' },
]); ]);
}); });
}); });

251
apps/outreach-worker/src/sources/nhdoj/parse-registry.ts
View File

@@ -1,55 +1,92 @@
/** /**
* PURE row reconstruction over positioned text extracted from the NHDOJ * PURE row reconstruction over positioned text extracted from the NHDOJ
* Charitable Trusts registry PDF (see `extract-pdf-text.ts` for the impure * Charitable Trusts registry PDF (see `extract-pdf-text.ts` for the impure
* layer that produces the input). No PDF or network dependency here — every * layer). No PDF or network dependency here — every function operates on
* function operates on plain `PositionedTextItem[][]` so tests can drive it * plain `PositionedTextItem[][]` so tests drive it with synthetic fixtures.
* with synthetic fixtures.
* *
* The registry renders as a 3-column table (organization name, city, * Layout (verified against the real PDF, "Registered Charities List",
* registration status). pdf.js gives us a flat bag of positioned glyph runs * updated 2026-07-08, 427 pages): an 8-column table —
* per page with no row/column structure, so reconstruction happens in two
* passes:
* *
* Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due
*
* with a legend line "G = Good Standing; X = Not in Good Standing;
* S = Suspended" under the letterhead, a repeated header row per page, and
* an "Updated: <date>" footer. Status is a single letter (G/X/S). The list
* includes out-of-state charities registered to solicit in NH, so State is
* carried through rather than assumed 'NH'.
*
* Reconstruction:
* 1. `reconstructRegistryRows` groups items into visual lines by y * 1. `reconstructRegistryRows` groups items into visual lines by y
* (tolerance ~2pt), buckets each line's items into columns by x * (~2pt tolerance), buckets each line's items into the 8 columns by x
* (boundaries inferred from the header row's token x-positions, or * (anchors inferred from the header row's token positions, or supplied
* supplied explicitly via `options.columnBoundaries` when a page's * via `options.columnAnchors`), and folds wrapped lines — a line with
* header doesn't repeat), and stitches multi-line org names back * no Reg. No. and no Status continues the previous row's name/address.
* together — a continuation line has text only in the name column. * 2. `normalizeRegistryRows` maps G/X/S onto the status enum and drops
* 2. `normalizeRegistryRows` maps the free-text status column onto the * artifact rows (legend, letterhead, page footers).
* tri-state the DB expects and drops anything that isn't really a data
* row (repeated header, page number, agency letterhead).
*/ */
import type { PositionedTextItem } from './extract-pdf-text.js'; import type { PositionedTextItem } from './extract-pdf-text.js';
export interface RegistryRow { export interface RegistryRow {
readonly registrationNumber: string | null;
readonly name: string; readonly name: string;
readonly address: string | null;
readonly city: string | null; readonly city: string | null;
readonly state: string | null;
readonly zip: string | null;
readonly status: string; readonly status: string;
readonly reportDue: string | null;
} }
export type NormalizedRegistrationStatus = 'good_standing' | 'lapsed' | 'unknown'; export type NormalizedRegistrationStatus =
| 'good_standing'
| 'lapsed'
| 'suspended'
| 'unknown';
export interface NormalizedRegistryRow { export interface NormalizedRegistryRow {
readonly registrationNumber: string | null;
readonly name: string; readonly name: string;
readonly city: string | null; readonly city: string | null;
readonly state: string | null;
readonly status: NormalizedRegistrationStatus; readonly status: NormalizedRegistrationStatus;
} }
/** Ascending x-anchors for the 8 columns, left edge of each. */
export type ColumnAnchors = readonly [
number, // Reg. No.
number, // Charity Name
number, // Address
number, // City
number, // State
number, // Zip
number, // Status
number, // Report Due
];
export interface ParseRegistryOptions { export interface ParseRegistryOptions {
/** Max y-distance (pt) between items considered part of the same visual line. Default 2. */ /** Max y-distance (pt) between items on the same visual line. Default 2. */
readonly yTolerance?: number; readonly yTolerance?: number;
/** /**
* Explicit x-position column starts `[nameStart, cityStart, statusStart]`, * Explicit column anchors. When omitted, inferred per page from that
* ascending. When omitted, boundaries are inferred per page from that * page's header row; pages without a header reuse the last-known anchors.
* page's header row (falling back to the most recently inferred/ provided
* boundaries for pages whose header doesn't repeat, e.g. continuation
* pages).
*/ */
readonly columnBoundaries?: readonly [number, number, number]; readonly columnAnchors?: ColumnAnchors;
} }
const DEFAULT_Y_TOLERANCE = 2; const DEFAULT_Y_TOLERANCE = 2;
/** Data cells sit within a few pt left of their header token's x. */
const COLUMN_X_SLACK = 4;
const HEADER_PATTERNS: readonly RegExp[] = [
/^reg\.?\s*no\.?$/i,
/^charity\s+name$/i,
/^address$/i,
/^city$/i,
/^state$/i,
/^zip$/i,
/^status$/i,
/^report\s+due$/i,
];
interface Line { interface Line {
readonly y: number; readonly y: number;
@@ -60,10 +97,7 @@ function collapseWhitespace(text: string): string {
return text.trim().replace(/\s+/g, ' '); return text.trim().replace(/\s+/g, ' ');
} }
/** /** Groups items into visual lines: top-to-bottom, left-to-right. */
* Groups a page's positioned items into visual lines (top-to-bottom by y,
* items within a line ordered left-to-right by x).
*/
function groupLines( function groupLines(
items: readonly PositionedTextItem[], items: readonly PositionedTextItem[],
yTolerance: number, yTolerance: number,
@@ -88,33 +122,29 @@ function groupLines(
return lines; return lines;
} }
/** Bucket an x-position into a column index given ascending column-start boundaries. */ function columnIndexForX(x: number, anchors: ColumnAnchors): number {
function columnIndexForX(x: number, boundaries: readonly number[]): number { for (let i = anchors.length - 1; i >= 0; i--) {
for (let i = boundaries.length - 1; i >= 0; i--) { if (x >= anchors[i]! - COLUMN_X_SLACK) return i;
if (x >= boundaries[i]!) return i;
} }
return 0; return 0;
} }
/** /**
* Looks for a line whose items spell out the three column headers * Finds the header row (all 8 column labels on one line) and returns each
* ("Organization"/"Name", "City", "Status") and returns their x-positions, * label's x-position as the column anchors.
* ascending — which is also left-to-right table order (name, city, status).
*/ */
function inferColumnBoundariesFromHeader( function inferColumnAnchorsFromHeader(
lines: readonly Line[], lines: readonly Line[],
): [number, number, number] | null { ): ColumnAnchors | null {
for (const line of lines) { for (const line of lines) {
const nameItem = line.items.find((i) => const anchors: number[] = [];
/organi[sz]ation|^name$/i.test(i.str.trim()), for (const pattern of HEADER_PATTERNS) {
); const hit = line.items.find((i) => pattern.test(i.str.trim()));
const cityItem = line.items.find((i) => /^city$/i.test(i.str.trim())); if (hit == null) break;
const statusItem = line.items.find((i) => /status/i.test(i.str.trim())); anchors.push(hit.x);
}
if (nameItem != null && cityItem != null && statusItem != null) { if (anchors.length === HEADER_PATTERNS.length) {
return [nameItem.x, cityItem.x, statusItem.x].sort( return [...anchors].sort((a, b) => a - b) as unknown as ColumnAnchors;
(a, b) => a - b,
) as [number, number, number];
} }
} }
return null; return null;
@@ -123,19 +153,23 @@ function inferColumnBoundariesFromHeader(
function isColumnHeaderLine(lineText: string): boolean { function isColumnHeaderLine(lineText: string): boolean {
const upper = lineText.toUpperCase(); const upper = lineText.toUpperCase();
return ( return (
(upper.includes('ORGANIZATION') || /\bNAME\b/.test(upper)) && upper.includes('CHARITY NAME') &&
upper.includes('CITY') && upper.includes('CITY') &&
upper.includes('STATUS') upper.includes('STATUS')
); );
} }
/** Page-number footers, repeated agency letterhead, and blank lines. */ /** Legend, letterhead, page footers, and blank lines. */
function isFooterOrArtifactLine(lineText: string): boolean { function isFooterOrArtifactLine(lineText: string): boolean {
const trimmed = lineText.trim(); const trimmed = lineText.trim();
if (trimmed === '') return true; if (trimmed === '') return true;
if (/^\d+$/.test(trimmed)) return true; if (/^\d+$/.test(trimmed)) return true;
if (/^page\s+\d+(\s+of\s+\d+)?$/i.test(trimmed)) return true; if (/^page\s+\d+(\s+of\s+\d+)?$/i.test(trimmed)) return true;
if (/department of justice/i.test(trimmed)) return true; if (/department of justice/i.test(trimmed)) return true;
if (/registered charities list/i.test(trimmed)) return true;
if (/charitable trusts unit/i.test(trimmed)) return true;
if (/=\s*good standing/i.test(trimmed)) return true;
if (/^updated:/i.test(trimmed)) return true;
return false; return false;
} }
@@ -144,22 +178,19 @@ function isHeaderOrFooterLine(lineText: string): boolean {
} }
/** /**
* Reconstructs `{ name, city, status }` rows from positioned text, one page * Reconstructs registry rows from positioned text, one page at a time.
* array at a time. Multi-line org names (a continuation line with nothing * Wrapped rows (no Reg. No., no Status) fold their name/address text back
* in the city/status columns) are folded back into the previous row. * into the previous row.
* *
* Throws if a page has data but no column boundaries can be determined * Throws if a page has data but no column anchors can be determined —
* (no header row found on any page so far, and none supplied) — that means * a layout change should fail loudly, not misparse silently.
* the registry's layout changed and silent misparsing is worse than a loud
* failure here.
*/ */
export function reconstructRegistryRows( export function reconstructRegistryRows(
pages: readonly (readonly PositionedTextItem[])[], pages: readonly (readonly PositionedTextItem[])[],
options: ParseRegistryOptions = {}, options: ParseRegistryOptions = {},
): RegistryRow[] { ): RegistryRow[] {
const yTolerance = options.yTolerance ?? DEFAULT_Y_TOLERANCE; const yTolerance = options.yTolerance ?? DEFAULT_Y_TOLERANCE;
let boundaries: readonly [number, number, number] | null = let anchors: ColumnAnchors | null = options.columnAnchors ?? null;
options.columnBoundaries ?? null;
const rows: RegistryRow[] = []; const rows: RegistryRow[] = [];
@@ -168,15 +199,15 @@ export function reconstructRegistryRows(
const lines = groupLines(pageItems, yTolerance); const lines = groupLines(pageItems, yTolerance);
if (options.columnBoundaries == null) { if (options.columnAnchors == null) {
const inferred = inferColumnBoundariesFromHeader(lines); const inferred = inferColumnAnchorsFromHeader(lines);
if (inferred != null) boundaries = inferred; if (inferred != null) anchors = inferred;
} }
if (boundaries == null) { if (anchors == null) {
throw new Error( throw new Error(
'reconstructRegistryRows: could not determine column boundaries ' + 'reconstructRegistryRows: could not determine column anchors ' +
'(no header row found and none provided via options.columnBoundaries)', '(no header row found and none provided via options.columnAnchors)',
); );
} }
@@ -184,60 +215,97 @@ export function reconstructRegistryRows(
const lineText = line.items.map((i) => i.str).join(' '); const lineText = line.items.map((i) => i.str).join(' ');
if (isHeaderOrFooterLine(lineText)) continue; if (isHeaderOrFooterLine(lineText)) continue;
const columns: [string[], string[], string[]] = [[], [], []]; const columns: string[][] = Array.from(
{ length: HEADER_PATTERNS.length },
() => [],
);
for (const item of line.items) { for (const item of line.items) {
const idx = columnIndexForX(item.x, boundaries); if (item.str.trim() === '') continue;
columns[idx as 0 | 1 | 2].push(item.str); columns[columnIndexForX(item.x, anchors)]!.push(item.str);
} }
const name = collapseWhitespace(columns[0].join(' ')); const cell = (i: number): string =>
const city = collapseWhitespace(columns[1].join(' ')); collapseWhitespace(columns[i]!.join(' '));
const status = collapseWhitespace(columns[2].join(' '));
if (name === '' && city === '' && status === '') continue; const registrationNumber = cell(0);
const name = cell(1);
const address = cell(2);
const city = cell(3);
const state = cell(4);
const zip = cell(5);
const status = cell(6);
const reportDue = cell(7);
// A line with text only in the name column continues the previous if (
// row's (wrapped) org name rather than starting a new row. [registrationNumber, name, address, city, state, zip, status, reportDue]
if (city === '' && status === '' && rows.length > 0) { .every((c) => c === '')
) {
continue;
}
// A wrapped row: no registry number and no status — its name/address
// text continues the previous row rather than starting a new one.
if (registrationNumber === '' && status === '' && rows.length > 0) {
const prev = rows[rows.length - 1]!; const prev = rows[rows.length - 1]!;
rows[rows.length - 1] = { rows[rows.length - 1] = {
...prev, ...prev,
name: collapseWhitespace(`${prev.name} ${name}`), name: collapseWhitespace(`${prev.name} ${name}`),
address:
address === ''
? prev.address
: collapseWhitespace(`${prev.address ?? ''} ${address}`),
}; };
continue; continue;
} }
rows.push({ name, city: city === '' ? null : city, status }); rows.push({
registrationNumber: registrationNumber === '' ? null : registrationNumber,
name,
address: address === '' ? null : address,
city: city === '' ? null : city,
state: state === '' ? null : state,
zip: zip === '' ? null : zip,
status,
reportDue: reportDue === '' ? null : reportDue,
});
} }
} }
return rows; return rows;
} }
const GOOD_STANDING_PATTERNS = [/good\s*standing/i, /\bcurrent\b/i, /\bactive\b/i]; /**
const LAPSED_PATTERNS = [ * Registry legend: G = Good Standing; X = Not in Good Standing;
/\blapsed\b/i, * S = Suspended. Phrase forms accepted as a fallback for older
/\bdelinquent\b/i, * republishes that spelled statuses out.
/\bsuspended\b/i, */
/\bexpired\b/i,
/\brevoked\b/i,
];
export function normalizeRegistrationStatus( export function normalizeRegistrationStatus(
status: string, status: string,
): NormalizedRegistrationStatus { ): NormalizedRegistrationStatus {
const trimmed = status.trim(); const trimmed = status.trim();
if (trimmed === '') return 'unknown'; if (trimmed === '') return 'unknown';
if (GOOD_STANDING_PATTERNS.some((p) => p.test(trimmed))) return 'good_standing'; if (/^g$/i.test(trimmed) || /good\s*standing/i.test(trimmed)) {
if (LAPSED_PATTERNS.some((p) => p.test(trimmed))) return 'lapsed'; // "Not in Good Standing" also contains the phrase — X handles the real
// file; this phrase fallback must not swallow the negated form.
if (/not\s+in\s+good/i.test(trimmed)) return 'lapsed';
return 'good_standing';
}
if (/^s$/i.test(trimmed) || /\bsuspended\b/i.test(trimmed)) return 'suspended';
if (
/^x$/i.test(trimmed) ||
/\blapsed\b/i.test(trimmed) ||
/\bdelinquent\b/i.test(trimmed) ||
/\bexpired\b/i.test(trimmed) ||
/\brevoked\b/i.test(trimmed)
) {
return 'lapsed';
}
return 'unknown'; return 'unknown';
} }
/** /**
* Maps raw rows onto the tri-state status the DB expects, collapses * Maps raw rows onto the DB's status enum, trims to the fields the org
* whitespace, and drops anything that isn't really a registrant (a * table stores, and drops anything that isn't really a registrant.
* repeated header row or letterhead line that slipped through row
* reconstruction as a degenerate one-column row).
*/ */
export function normalizeRegistryRows( export function normalizeRegistryRows(
rows: readonly RegistryRow[], rows: readonly RegistryRow[],
@@ -249,9 +317,14 @@ export function normalizeRegistryRows(
if (name === '' || isHeaderOrFooterLine(name)) continue; if (name === '' || isHeaderOrFooterLine(name)) continue;
const city = row.city == null ? null : collapseWhitespace(row.city); const city = row.city == null ? null : collapseWhitespace(row.city);
const state =
row.state == null ? null : collapseWhitespace(row.state).toUpperCase();
normalized.push({ normalized.push({
registrationNumber: row.registrationNumber,
name, name,
city: city === '' ? null : city, city: city === '' ? null : city,
state: state === '' ? null : state,
status: normalizeRegistrationStatus(row.status), status: normalizeRegistrationStatus(row.status),
}); });
} }

7
apps/outreach-worker/src/sources/propublica/client.ts
View File

@@ -62,6 +62,13 @@ export async function searchOrganizations(
url.searchParams.set('state[id]', state); url.searchParams.set('state[id]', state);
const res = await fetch(url); const res = await fetch(url);
if (res.status === 404) {
// Nonprofit Explorer quirk (verified 2026-07-16): a filtered search
// with zero hits responds 404, not 200-with-empty-list. Zero hits is a
// legitimate "no match" — the org resolves as unresolved, not failed.
await wait(options.politenessDelayMs ?? DEFAULT_POLITENESS_DELAY_MS);
return { organizations: [] };
}
if (!res.ok) { if (!res.ok) {
throw new Error( throw new Error(
`ProPublica search failed for "${name}" (${state}): ${res.status} ${res.statusText}`, `ProPublica search failed for "${name}" (${state}): ${res.status} ${res.statusText}`,

19
apps/outreach-worker/src/workflows/enrich-orgs.ts
View File

@@ -181,6 +181,7 @@ async function runEnrichOrgs(): Promise<void> {
const g = globalThis as unknown as { const g = globalThis as unknown as {
__outreachEnrichOrgsRegistered?: boolean; __outreachEnrichOrgsRegistered?: boolean;
__outreachEnrichOrgsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
}; };
if (!g.__outreachEnrichOrgsRegistered) { if (!g.__outreachEnrichOrgsRegistered) {
@@ -197,10 +198,26 @@ if (!g.__outreachEnrichOrgsRegistered) {
// Must be registered as BOTH a workflow and a scheduled function, // Must be registered as BOTH a workflow and a scheduled function,
// referencing the same function object — see module doc comment. // referencing the same function object — see module doc comment.
DBOS.registerWorkflow(enrichOrgs, { name: 'enrichOrgs' }); g.__outreachEnrichOrgsHandle = DBOS.registerWorkflow(enrichOrgs, {
name: 'enrichOrgs',
});
DBOS.registerScheduled(enrichOrgs, { DBOS.registerScheduled(enrichOrgs, {
crontab: '0 5 * * *', crontab: '0 5 * * *',
name: 'enrichOrgs', name: 'enrichOrgs',
mode: SchedulerMode.ExactlyOncePerInterval, mode: SchedulerMode.ExactlyOncePerInterval,
}); });
} }
/**
* Starts one durable run of this workflow immediately through DBOS —
* the exact production path (workflow + checkpointed steps), used by
* `run-once.ts` for supervised/manual passes. Requires deps injected and
* `DBOS.launch()` completed.
*/
export function runEnrichOrgsNow(): Promise<void> {
const handle = g.__outreachEnrichOrgsHandle;
if (handle == null) {
throw new Error('enrichOrgs is not registered; was this module imported before DBOS.launch()?');
}
return handle(new Date(), new Date());
}

19
apps/outreach-worker/src/workflows/expire-grants.ts
View File

@@ -47,6 +47,7 @@ async function runExpireGrants(): Promise<void> {
const g = globalThis as unknown as { const g = globalThis as unknown as {
__outreachExpireGrantsRegistered?: boolean; __outreachExpireGrantsRegistered?: boolean;
__outreachExpireGrantsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
}; };
if (!g.__outreachExpireGrantsRegistered) { if (!g.__outreachExpireGrantsRegistered) {
@@ -61,10 +62,26 @@ if (!g.__outreachExpireGrantsRegistered) {
} }
}; };
DBOS.registerWorkflow(expireGrants, { name: 'expireGrants' }); g.__outreachExpireGrantsHandle = DBOS.registerWorkflow(expireGrants, {
name: 'expireGrants',
});
DBOS.registerScheduled(expireGrants, { DBOS.registerScheduled(expireGrants, {
crontab: '0 * * * *', crontab: '0 * * * *',
name: 'expireGrants', name: 'expireGrants',
mode: SchedulerMode.ExactlyOncePerInterval, mode: SchedulerMode.ExactlyOncePerInterval,
}); });
} }
/**
* Starts one durable run of this workflow immediately through DBOS —
* the exact production path (workflow + checkpointed steps), used by
* `run-once.ts` for supervised/manual passes. Requires deps injected and
* `DBOS.launch()` completed.
*/
export function runExpireGrantsNow(): Promise<void> {
const handle = g.__outreachExpireGrantsHandle;
if (handle == null) {
throw new Error('expireGrants is not registered; was this module imported before DBOS.launch()?');
}
return handle(new Date(), new Date());
}

19
apps/outreach-worker/src/workflows/ingest-grants.ts
View File

@@ -154,6 +154,7 @@ async function runIngestGrants(): Promise<void> {
const g = globalThis as unknown as { const g = globalThis as unknown as {
__outreachIngestGrantsRegistered?: boolean; __outreachIngestGrantsRegistered?: boolean;
__outreachIngestGrantsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
}; };
if (!g.__outreachIngestGrantsRegistered) { if (!g.__outreachIngestGrantsRegistered) {
@@ -170,10 +171,26 @@ if (!g.__outreachIngestGrantsRegistered) {
// Must be registered as BOTH a workflow and a scheduled function, // Must be registered as BOTH a workflow and a scheduled function,
// referencing the same function object — see module doc comment. // referencing the same function object — see module doc comment.
DBOS.registerWorkflow(ingestGrants, { name: 'ingestGrants' }); g.__outreachIngestGrantsHandle = DBOS.registerWorkflow(ingestGrants, {
name: 'ingestGrants',
});
DBOS.registerScheduled(ingestGrants, { DBOS.registerScheduled(ingestGrants, {
crontab: '0 3 * * *', crontab: '0 3 * * *',
name: 'ingestGrants', name: 'ingestGrants',
mode: SchedulerMode.ExactlyOncePerInterval, mode: SchedulerMode.ExactlyOncePerInterval,
}); });
} }
/**
* Starts one durable run of this workflow immediately through DBOS —
* the exact production path (workflow + checkpointed steps), used by
* `run-once.ts` for supervised/manual passes. Requires deps injected and
* `DBOS.launch()` completed.
*/
export function runIngestGrantsNow(): Promise<void> {
const handle = g.__outreachIngestGrantsHandle;
if (handle == null) {
throw new Error('ingestGrants is not registered; was this module imported before DBOS.launch()?');
}
return handle(new Date(), new Date());
}

57
apps/outreach-worker/src/workflows/ingest-nhdoj-orgs.ts
View File

@@ -50,12 +50,29 @@ function getIngestNhdojOrgsDeps(): IngestNhdojOrgsDeps {
} }
async function fetchNhdojRegistryPdf(url: string): Promise<Uint8Array> { async function fetchNhdojRegistryPdf(url: string): Promise<Uint8Array> {
const res = await fetch(url); // mm.nh.gov sits behind Akamai bot detection: bare curl/default clients
// get a 403 "Access Denied" HTML page. Node's fetch passes with
// browser-like headers (verified 2026-07-16).
const res = await fetch(url, {
headers: {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
Accept: 'application/pdf,*/*',
},
});
if (!res.ok) { if (!res.ok) {
throw new Error( throw new Error(
`fetchNhdojRegistryPdf: ${url} responded ${res.status} ${res.statusText}`, `fetchNhdojRegistryPdf: ${url} responded ${res.status} ${res.statusText}`,
); );
} }
const contentType = res.headers.get('content-type') ?? '';
if (!contentType.includes('pdf')) {
// Akamai serves its denial as 200 text/html through some paths — treat
// a non-PDF body as a failure rather than feeding HTML to pdfjs.
throw new Error(
`fetchNhdojRegistryPdf: expected a PDF but got content-type "${contentType}" from ${url}`,
);
}
const buf = await res.arrayBuffer(); const buf = await res.arrayBuffer();
return new Uint8Array(buf); return new Uint8Array(buf);
} }
@@ -72,6 +89,10 @@ async function upsertRegistryOrg(
return serverUpsertOrgFromRegistry(db, { return serverUpsertOrgFromRegistry(db, {
name: row.name, name: row.name,
city: row.city, city: row.city,
// The registry includes out-of-state charities registered to solicit
// in NH — carry their real state so the geography gate stays honest.
state: row.state,
registrationNumber: row.registrationNumber,
registrationStatus: row.status, registrationStatus: row.status,
sourceRegistry: SOURCE_REGISTRY, sourceRegistry: SOURCE_REGISTRY,
}); });
@@ -85,18 +106,25 @@ const upsertRegistryOrgStep = DBOS.registerStep(upsertRegistryOrg, {
async function runIngestNhdojOrgs(): Promise<void> { async function runIngestNhdojOrgs(): Promise<void> {
const { db } = getIngestNhdojOrgsDeps(); const { db } = getIngestNhdojOrgsDeps();
const pdfPath = process.env.NHDOJ_REGISTRY_PDF_PATH;
const pdfUrl = process.env.NHDOJ_REGISTRY_PDF_URL; const pdfUrl = process.env.NHDOJ_REGISTRY_PDF_URL;
if (pdfUrl == null || pdfUrl.trim() === '') {
let pdfBytes: Uint8Array;
if (pdfPath != null && pdfPath.trim() !== '') {
// Local-file escape hatch for supervised runs and Akamai outages.
const { readFile } = await import('node:fs/promises');
pdfBytes = new Uint8Array(await readFile(pdfPath));
} else if (pdfUrl != null && pdfUrl.trim() !== '') {
pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl);
} else {
// The registry PDF's URL changes whenever NHDOJ republishes it (no // The registry PDF's URL changes whenever NHDOJ republishes it (no
// stable endpoint) — a hard failure here would page on-call for a // stable endpoint) — a hard failure here would page on-call for a
// config gap rather than a real outage, so skip quietly instead. // config gap rather than a real outage, so skip quietly instead.
console.warn( console.warn(
'[ingest-nhdoj-orgs] NHDOJ_REGISTRY_PDF_URL is not set; skipping this pass.', '[ingest-nhdoj-orgs] neither NHDOJ_REGISTRY_PDF_PATH nor NHDOJ_REGISTRY_PDF_URL is set; skipping this pass.',
); );
return; return;
} }
const pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl);
const pages = await extractPositionedText(pdfBytes); const pages = await extractPositionedText(pdfBytes);
const rawRows = reconstructRegistryRows(pages); const rawRows = reconstructRegistryRows(pages);
const normalizedRows = normalizeRegistryRows(rawRows); const normalizedRows = normalizeRegistryRows(rawRows);
@@ -116,6 +144,7 @@ async function runIngestNhdojOrgs(): Promise<void> {
const g = globalThis as unknown as { const g = globalThis as unknown as {
__outreachIngestNhdojOrgsRegistered?: boolean; __outreachIngestNhdojOrgsRegistered?: boolean;
__outreachIngestNhdojOrgsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
}; };
if (!g.__outreachIngestNhdojOrgsRegistered) { if (!g.__outreachIngestNhdojOrgsRegistered) {
@@ -132,10 +161,26 @@ if (!g.__outreachIngestNhdojOrgsRegistered) {
// Must be registered as BOTH a workflow and a scheduled function, // Must be registered as BOTH a workflow and a scheduled function,
// referencing the same function object — see module doc comment. // referencing the same function object — see module doc comment.
DBOS.registerWorkflow(ingestNhdojOrgs, { name: 'ingestNhdojOrgs' }); g.__outreachIngestNhdojOrgsHandle = DBOS.registerWorkflow(ingestNhdojOrgs, {
name: 'ingestNhdojOrgs',
});
DBOS.registerScheduled(ingestNhdojOrgs, { DBOS.registerScheduled(ingestNhdojOrgs, {
crontab: '0 4 1 * *', crontab: '0 4 1 * *',
name: 'ingestNhdojOrgs', name: 'ingestNhdojOrgs',
mode: SchedulerMode.ExactlyOncePerInterval, mode: SchedulerMode.ExactlyOncePerInterval,
}); });
} }
/**
* Starts one durable run of this workflow immediately through DBOS —
* the exact production path (workflow + checkpointed steps), used by
* `run-once.ts` for supervised/manual passes. Requires deps injected and
* `DBOS.launch()` completed.
*/
export function runIngestNhdojOrgsNow(): Promise<void> {
const handle = g.__outreachIngestNhdojOrgsHandle;
if (handle == null) {
throw new Error('ingestNhdojOrgs is not registered; was this module imported before DBOS.launch()?');
}
return handle(new Date(), new Date());
}

19
apps/outreach-worker/src/workflows/ingest-pnd-rss.ts
View File

@@ -88,6 +88,7 @@ async function runIngestPndRss(): Promise<void> {
const g = globalThis as unknown as { const g = globalThis as unknown as {
__outreachIngestPndRssRegistered?: boolean; __outreachIngestPndRssRegistered?: boolean;
__outreachIngestPndRssHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
}; };
if (!g.__outreachIngestPndRssRegistered) { if (!g.__outreachIngestPndRssRegistered) {
@@ -104,10 +105,26 @@ if (!g.__outreachIngestPndRssRegistered) {
// Must be registered as BOTH a workflow and a scheduled function, // Must be registered as BOTH a workflow and a scheduled function,
// referencing the same function object — see module doc comment. // referencing the same function object — see module doc comment.
DBOS.registerWorkflow(ingestPndRss, { name: 'ingestPndRss' }); g.__outreachIngestPndRssHandle = DBOS.registerWorkflow(ingestPndRss, {
name: 'ingestPndRss',
});
DBOS.registerScheduled(ingestPndRss, { DBOS.registerScheduled(ingestPndRss, {
crontab: '30 3 * * *', crontab: '30 3 * * *',
name: 'ingestPndRss', name: 'ingestPndRss',
mode: SchedulerMode.ExactlyOncePerInterval, mode: SchedulerMode.ExactlyOncePerInterval,
}); });
} }
/**
* Starts one durable run of this workflow immediately through DBOS —
* the exact production path (workflow + checkpointed steps), used by
* `run-once.ts` for supervised/manual passes. Requires deps injected and
* `DBOS.launch()` completed.
*/
export function runIngestPndRssNow(): Promise<void> {
const handle = g.__outreachIngestPndRssHandle;
if (handle == null) {
throw new Error('ingestPndRss is not registered; was this module imported before DBOS.launch()?');
}
return handle(new Date(), new Date());
}

14
docs/features/ingestion.md
View File

@@ -47,6 +47,13 @@ Grant upserts key on `grants.source_url`; org registry upserts key on case-insen
### Philanthropy News Digest RSS ### Philanthropy News Digest RSS
> **Status (2026-07-16): feed retired upstream.** Every historical feed path
> (`/feeds/rfps.rss` and variants) now returns the site's HTML shell — PND is
> a Next.js app with client-side data and no `<link rel=alternate>` feeds.
> The workflow runs and upserts zero rows. Rework candidate: drive their
> internal JSON API / `__NEXT_DATA__` instead. Low priority — PND was "cheap
> incremental coverage"; Grants.gov + NH state sources carry the corpus.
Nightly ingestion of the Philanthropy News Digest "RFPs" RSS feed (`https://philanthropynewsdigest.org/feeds/rfps.rss`, overridable via `PND_RFP_FEED_URL`). Nightly ingestion of the Philanthropy News Digest "RFPs" RSS feed (`https://philanthropynewsdigest.org/feeds/rfps.rss`, overridable via `PND_RFP_FEED_URL`).
- **Client** — `src/sources/pnd-rss/client.ts`: `fetchPndRfpFeed(feedUrl)` does a plain `fetch`, throwing on any non-2xx response. - **Client** — `src/sources/pnd-rss/client.ts`: `fetchPndRfpFeed(feedUrl)` does a plain `fetch`, throwing on any non-2xx response.
@@ -94,3 +101,10 @@ Monthly re-scan of the NH Department of Justice Charitable Trusts Unit's registr
// ... // ...
setIngestNhdojOrgsDeps({ db }); setIngestNhdojOrgsDeps({ db });
``` ```
## First-run field findings (2026-07-16)
- **NHDOJ registry**: 8-column layout (`Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due`), single-letter statuses (G/X/S legend), includes out-of-state charities registered to solicit in NH. Parsed 13,709 registrants → 13,632 orgs (6,266 NH). `Reg. No.` is the stable upsert key (`orgs.registration_number`). mm.nh.gov sits behind Akamai: bare curl gets 403; Node fetch with browser-like headers passes (client sends them). `NHDOJ_REGISTRY_PDF_PATH` overrides the URL for supervised runs.
- **ProPublica**: a state-scoped search with zero hits returns **404**, not an empty list — the client maps 404 → no candidates (org resolves `unresolved`). Before the fix this tripped the 20% systemic-failure breaker.
- **Grants.gov**: first pass ingested 200 opportunities (search cap 1000 / detail cap 200); the hourly expiry sweep correctly expired a same-day deadline immediately.
- **Enrichment queue** prioritizes NH + good-standing orgs before the out-of-state tail; ~60% of NH orgs resolve an EIN per batch, the rest record an all-null attempt.

3
packages/outreach-core/drizzle/server/1784231856_nhdoj-registry-fields.sql Normal file
View File

@@ -0,0 +1,3 @@
ALTER TYPE "public"."registration_status" ADD VALUE IF NOT EXISTS 'suspended' BEFORE 'unknown';--> statement-breakpoint
ALTER TABLE "orgs" ADD COLUMN "registration_number" text;--> statement-breakpoint
CREATE UNIQUE INDEX "idx_orgs_registry_reg_no" ON "orgs" USING btree ("source_registry","registration_number") WHERE "orgs"."source_registry" IS NOT NULL AND "orgs"."registration_number" IS NOT NULL;

1149
packages/outreach-core/drizzle/server/meta/1784231856_snapshot.json Normal file
View File

File diff suppressed because it is too large Load Diff

7
packages/outreach-core/drizzle/server/meta/_journal.json
View File

@@ -8,6 +8,13 @@
"when": 1784214387212, "when": 1784214387212,
"tag": "1784214387_initial-schema", "tag": "1784214387_initial-schema",
"breakpoints": true "breakpoints": true
},
{
"idx": 1,
"version": "7",
"when": 1784231856865,
"tag": "1784231856_nhdoj-registry-fields",
"breakpoints": true
} }
] ]
} }

9
packages/outreach-core/src/db/schema.ts
View File

@@ -43,6 +43,7 @@ export const grantStatusEnum = pgEnum('grant_status', [
export const registrationStatusEnum = pgEnum('registration_status', [ export const registrationStatusEnum = pgEnum('registration_status', [
'good_standing', 'good_standing',
'lapsed', 'lapsed',
'suspended',
'unknown', 'unknown',
]); ]);
@@ -174,6 +175,9 @@ export const orgs = pgTable(
nteeCode: text('ntee_code'), nteeCode: text('ntee_code'),
totalRevenue: integer('total_revenue'), totalRevenue: integer('total_revenue'),
fiscalYearEnd: text('fiscal_year_end'), fiscalYearEnd: text('fiscal_year_end'),
// Registry-assigned identifier (e.g. NHDOJ "Reg. No.") — stable across
// republishes, unlike org names; the preferred upsert key when present.
registrationNumber: text('registration_number'),
registrationStatus: registrationStatusEnum('registration_status') registrationStatus: registrationStatusEnum('registration_status')
.notNull() .notNull()
.default('unknown'), .default('unknown'),
@@ -186,6 +190,11 @@ export const orgs = pgTable(
// Partial unique index: EIN is unique when present, but many small // Partial unique index: EIN is unique when present, but many small
// orgs in early ingestion won't have one resolved yet. // orgs in early ingestion won't have one resolved yet.
uniqueIndex('idx_orgs_ein').on(t.ein).where(sql`${t.ein} IS NOT NULL`), uniqueIndex('idx_orgs_ein').on(t.ein).where(sql`${t.ein} IS NOT NULL`),
uniqueIndex('idx_orgs_registry_reg_no')
.on(t.sourceRegistry, t.registrationNumber)
.where(
sql`${t.sourceRegistry} IS NOT NULL AND ${t.registrationNumber} IS NOT NULL`,
),
index('idx_orgs_icp_band').on(t.icpBand), index('idx_orgs_icp_band').on(t.icpBand),
index('idx_orgs_state').on(t.state), index('idx_orgs_state').on(t.state),
], ],

59
packages/outreach-core/src/orgs/actions/upsert-org-from-registry.server.ts
View File

@@ -6,8 +6,10 @@ import { schema } from '#~/db/db.js';
export interface RegistryOrgInput { export interface RegistryOrgInput {
readonly name: string; readonly name: string;
readonly city: string | null; readonly city: string | null;
readonly state?: string; readonly state?: string | null;
readonly registrationStatus: 'good_standing' | 'lapsed' | 'unknown'; /** Registry-assigned id (e.g. NHDOJ Reg. No.) — preferred identity when present. */
readonly registrationNumber?: string | null;
readonly registrationStatus: 'good_standing' | 'lapsed' | 'suspended' | 'unknown';
readonly sourceRegistry: string; readonly sourceRegistry: string;
} }
@@ -18,12 +20,17 @@ export interface RegistryUpsertResult {
/** /**
* Upserts an org discovered in a charity registry re-scan (e.g. the NHDOJ * Upserts an org discovered in a charity registry re-scan (e.g. the NHDOJ
* Charitable Trusts list). Registries carry no EIN, so identity is the * Charitable Trusts list).
* case-insensitive (name, city, state) triple — good enough for a *
* single-state registry where the same legal name in the same city is the * Identity, in preference order:
* same org. Existing rows get their registration status refreshed (orgs * 1. (sourceRegistry, registrationNumber) — the registry's own stable id;
* lapse and re-register); enrichment fields (EIN, revenue, NTEE) are never * survives renames and relocations. Name/city/state are refreshed from
* touched here. * the incoming row on this path.
* 2. Case-insensitive (name, city, state) — fallback for registries (or
* historical rows) without a registration number. Only status and
* provenance are refreshed on this path.
*
* Enrichment fields (EIN, revenue, NTEE) are never touched here.
* *
* Returns `inserted: true` for brand-new registrants — a segment the * Returns `inserted: true` for brand-new registrants — a segment the
* pipeline treats specially (often actively seeking first-time funding). * pipeline treats specially (often actively seeking first-time funding).
@@ -33,8 +40,37 @@ export async function serverUpsertOrgFromRegistry(
org: RegistryOrgInput, org: RegistryOrgInput,
): Promise<RegistryUpsertResult> { ): Promise<RegistryUpsertResult> {
const state = org.state ?? 'NH'; const state = org.state ?? 'NH';
const registrationNumber = org.registrationNumber ?? null;
const existing = await db if (registrationNumber != null) {
const byRegNo = await db
.select({ id: schema.orgs.id })
.from(schema.orgs)
.where(
and(
eq(schema.orgs.sourceRegistry, org.sourceRegistry),
eq(schema.orgs.registrationNumber, registrationNumber),
),
)
.limit(1);
const found = byRegNo[0];
if (found != null) {
await db
.update(schema.orgs)
.set({
name: org.name,
city: org.city,
state,
registrationStatus: org.registrationStatus,
updatedAt: sql`now()`,
})
.where(eq(schema.orgs.id, found.id));
return { id: found.id, inserted: false };
}
}
const byName = await db
.select({ id: schema.orgs.id }) .select({ id: schema.orgs.id })
.from(schema.orgs) .from(schema.orgs)
.where( .where(
@@ -48,13 +84,15 @@ export async function serverUpsertOrgFromRegistry(
) )
.limit(1); .limit(1);
const found = existing[0]; const found = byName[0];
if (found != null) { if (found != null) {
await db await db
.update(schema.orgs) .update(schema.orgs)
.set({ .set({
registrationStatus: org.registrationStatus, registrationStatus: org.registrationStatus,
sourceRegistry: org.sourceRegistry, sourceRegistry: org.sourceRegistry,
// Adopt the registry id so future re-scans match on the stable key.
registrationNumber,
updatedAt: sql`now()`, updatedAt: sql`now()`,
}) })
.where(eq(schema.orgs.id, found.id)); .where(eq(schema.orgs.id, found.id));
@@ -67,6 +105,7 @@ export async function serverUpsertOrgFromRegistry(
name: org.name, name: org.name,
city: org.city, city: org.city,
state, state,
registrationNumber,
registrationStatus: org.registrationStatus, registrationStatus: org.registrationStatus,
sourceRegistry: org.sourceRegistry, sourceRegistry: org.sourceRegistry,
}) })

10
packages/outreach-core/src/orgs/queries/list-orgs-needing-enrichment.server.ts
View File

@@ -1,4 +1,4 @@
import { and, asc, isNull } from 'drizzle-orm'; import { and, asc, isNull, sql } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js'; import { schema } from '#~/db/db.js';
@@ -31,6 +31,12 @@ export async function serverListOrgsNeedingEnrichment(
}) })
.from(schema.orgs) .from(schema.orgs)
.where(and(isNull(schema.orgs.ein), isNull(schema.orgs.totalRevenue))) .where(and(isNull(schema.orgs.ein), isNull(schema.orgs.totalRevenue)))
.orderBy(asc(schema.orgs.updatedAt)) // NH good-standing orgs are the ICP — enrich them before the long tail
// of out-of-state registrants; oldest-touched first within each tier.
.orderBy(
sql`(${schema.orgs.state} = 'NH') DESC`,
sql`(${schema.orgs.registrationStatus} = 'good_standing') DESC`,
asc(schema.orgs.updatedAt),
)
.limit(limit); .limit(limit);
} }