fix(ingestion): survive first contact with real data sources
NHDOJ: parser rebuilt for the real 8-column registry layout (Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due) with single-letter G/X/S statuses; Reg. No. is the stable upsert key (new orgs.registration_number column + partial unique index, enum gains 'suspended' via idempotent ADD VALUE); out-of-state registrants keep their real state. Akamai-safe fetch headers + NHDOJ_REGISTRY_PDF_PATH local-file override. ProPublica: zero-hit state-scoped searches return 404, not an empty list — map to no-candidates instead of failure (tripped the systemic- failure breaker at 60/200 on first contact). Enrichment queue now prioritizes NH good-standing orgs over the out-of-state tail. PND: feed retired upstream (HTML shell on every historical path) — documented as rework candidate, low priority. run-once.ts: supervised one-off runner through the durable DBOS handles (workflow modules now export run*Now accessors); drop the double pool.end() after DBOS.shutdown(). First supervised run: 200 Grants.gov opportunities (1 auto-expired), 13,632 orgs from the 427-page registry, enrichment at failed=0 with 121/200 EIN resolution in the NH-priority batch. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
@@ -16,6 +16,9 @@ GCP_SERVICE_ACCOUNT_KEY_PATH=./secrets/gcp-service-account.json
|
||||
# NHDOJ Charitable Trusts registry PDF — the URL changes whenever NHDOJ
|
||||
# republishes (roughly monthly). When unset, the monthly ingestNhdojOrgs
|
||||
# workflow logs a warning and no-ops instead of failing.
|
||||
# Local-file override for supervised runs / Akamai outages (takes
|
||||
# precedence over the URL):
|
||||
# NHDOJ_REGISTRY_PDF_PATH=./secrets/registered-charities.pdf
|
||||
# NHDOJ_REGISTRY_PDF_URL=https://www.doj.nh.gov/.../charitable-trusts-registry.pdf
|
||||
|
||||
# Optional override for the Philanthropy News Digest RFP feed (defaults to
|
||||
|
||||
120
apps/outreach-worker/src/run-once.ts
Normal file
120
apps/outreach-worker/src/run-once.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* Supervised one-off workflow runner.
|
||||
*
|
||||
* DATABASE_URL=... node --import tsx/esm src/run-once.ts <workflow> [...]
|
||||
*
|
||||
* where <workflow> is one or more of: ingestGrants, ingestPndRss,
|
||||
* ingestNhdojOrgs, enrichOrgs, expireGrants — or `all` for the standard
|
||||
* first-run order (grants → pnd → nhdoj → expire → enrich).
|
||||
*
|
||||
* Boots exactly like main.ts (same deps injection, same DBOS launch — see
|
||||
* that file's boot-order comment), runs the requested workflows through
|
||||
* their durable DBOS handles sequentially, then shuts down. Scheduled crons
|
||||
* ARE active while this process lives; runs are short enough that this
|
||||
* doesn't matter in practice.
|
||||
*/
|
||||
import { DBOS } from '@dbos-inc/dbos-sdk';
|
||||
import { DrizzleDataSource } from '@dbos-inc/drizzle-datasource';
|
||||
import { schema } from '@novelpad/outreach-core';
|
||||
import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
import pg from 'pg';
|
||||
|
||||
import { runEnrichOrgsNow, setEnrichOrgsDeps } from './workflows/enrich-orgs.js';
|
||||
import {
|
||||
runExpireGrantsNow,
|
||||
setExpireGrantsDeps,
|
||||
} from './workflows/expire-grants.js';
|
||||
import {
|
||||
runIngestGrantsNow,
|
||||
setIngestGrantsDeps,
|
||||
} from './workflows/ingest-grants.js';
|
||||
import {
|
||||
runIngestNhdojOrgsNow,
|
||||
setIngestNhdojOrgsDeps,
|
||||
} from './workflows/ingest-nhdoj-orgs.js';
|
||||
import {
|
||||
runIngestPndRssNow,
|
||||
setIngestPndRssDeps,
|
||||
} from './workflows/ingest-pnd-rss.js';
|
||||
|
||||
const RUNNERS: Record<string, () => Promise<void>> = {
|
||||
ingestGrants: runIngestGrantsNow,
|
||||
ingestPndRss: runIngestPndRssNow,
|
||||
ingestNhdojOrgs: runIngestNhdojOrgsNow,
|
||||
enrichOrgs: runEnrichOrgsNow,
|
||||
expireGrants: runExpireGrantsNow,
|
||||
};
|
||||
|
||||
const FIRST_RUN_ORDER = [
|
||||
'ingestGrants',
|
||||
'ingestPndRss',
|
||||
'ingestNhdojOrgs',
|
||||
'expireGrants',
|
||||
'enrichOrgs',
|
||||
];
|
||||
|
||||
if (process.env.DATABASE_URL == null) {
|
||||
throw new Error('run-once: DATABASE_URL is required');
|
||||
}
|
||||
|
||||
const requested = process.argv.slice(2);
|
||||
const names = requested.includes('all') ? FIRST_RUN_ORDER : requested;
|
||||
if (names.length === 0 || names.some((n) => RUNNERS[n] == null)) {
|
||||
console.error(
|
||||
`Usage: run-once.ts <workflow...>\nKnown workflows: ${Object.keys(RUNNERS).join(', ')}, all`,
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const { Pool } = pg;
|
||||
const dbosClientConfig = {
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
application_name: 'helmdocs-outreach-run-once',
|
||||
} satisfies pg.ClientConfig;
|
||||
const pool = new Pool({
|
||||
...dbosClientConfig,
|
||||
max: Number(process.env.PG_POOL_MAX ?? 20),
|
||||
});
|
||||
const db = drizzle(pool, { schema });
|
||||
|
||||
async function main() {
|
||||
await DrizzleDataSource.initializeDBOSSchema(dbosClientConfig);
|
||||
|
||||
setIngestGrantsDeps({ db });
|
||||
setExpireGrantsDeps({ db });
|
||||
setIngestPndRssDeps({ db });
|
||||
setIngestNhdojOrgsDeps({ db });
|
||||
setEnrichOrgsDeps({ db });
|
||||
|
||||
DBOS.setConfig({
|
||||
name: 'helmdocs-outreach-worker',
|
||||
systemDatabasePool: pool,
|
||||
runAdminServer: false,
|
||||
});
|
||||
await DBOS.launch();
|
||||
|
||||
let failed = false;
|
||||
for (const name of names) {
|
||||
console.log(`\n=== run-once: ${name} ===`);
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
await RUNNERS[name]!();
|
||||
console.log(
|
||||
`=== run-once: ${name} OK in ${((Date.now() - startedAt) / 1000).toFixed(1)}s ===`,
|
||||
);
|
||||
} catch (err) {
|
||||
failed = true;
|
||||
console.error(`=== run-once: ${name} FAILED ===`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// DBOS.shutdown() ends the pool we handed it via systemDatabasePool —
|
||||
// a second pool.end() here throws "Called end on pool more than once".
|
||||
await DBOS.shutdown();
|
||||
process.exit(failed ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[run-once] fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -5,236 +5,303 @@ import {
|
||||
normalizeRegistrationStatus,
|
||||
normalizeRegistryRows,
|
||||
reconstructRegistryRows,
|
||||
type RegistryRow,
|
||||
} from './parse-registry.js';
|
||||
|
||||
/** Column x-starts used across fixtures: name @50, city @300, status @450. */
|
||||
const NAME_X = 50;
|
||||
const CITY_X = 300;
|
||||
const STATUS_X = 450;
|
||||
// x-anchors observed in the real PDF (2026-07-08 republish).
|
||||
const X = {
|
||||
regNo: 17,
|
||||
name: 70,
|
||||
address: 380,
|
||||
city: 627,
|
||||
state: 710,
|
||||
zip: 748,
|
||||
status: 820,
|
||||
reportDue: 891,
|
||||
} as const;
|
||||
|
||||
function item(str: string, x: number, y: number): PositionedTextItem {
|
||||
return { str, x, y };
|
||||
}
|
||||
|
||||
function headerRow(y: number): PositionedTextItem[] {
|
||||
function headerLine(y: number): PositionedTextItem[] {
|
||||
return [
|
||||
item('Organization', NAME_X, y),
|
||||
item('City', CITY_X, y),
|
||||
item('Status', STATUS_X, y),
|
||||
item('Reg. No.', X.regNo, y),
|
||||
item('Charity Name', X.name, y),
|
||||
item('Address', X.address, y),
|
||||
item('City', X.city, y),
|
||||
item('State', X.state, y),
|
||||
item('Zip', X.zip, y),
|
||||
item('Status', X.status, y),
|
||||
item('Report Due', X.reportDue, y),
|
||||
];
|
||||
}
|
||||
|
||||
function dataLine(
|
||||
y: number,
|
||||
cells: {
|
||||
regNo?: string;
|
||||
name?: string;
|
||||
address?: string;
|
||||
city?: string;
|
||||
state?: string;
|
||||
zip?: string;
|
||||
status?: string;
|
||||
reportDue?: string;
|
||||
},
|
||||
): PositionedTextItem[] {
|
||||
const out: PositionedTextItem[] = [];
|
||||
if (cells.regNo) out.push(item(cells.regNo, X.regNo, y));
|
||||
if (cells.name) out.push(item(cells.name, X.name + 2, y));
|
||||
if (cells.address) out.push(item(cells.address, X.address, y));
|
||||
if (cells.city) out.push(item(cells.city, X.city, y));
|
||||
if (cells.state) out.push(item(cells.state, X.state, y));
|
||||
if (cells.zip) out.push(item(cells.zip, X.zip, y));
|
||||
if (cells.status) out.push(item(cells.status, X.status, y));
|
||||
if (cells.reportDue) out.push(item(cells.reportDue, X.reportDue, y));
|
||||
return out;
|
||||
}
|
||||
|
||||
function letterheadAndLegend(): PositionedTextItem[] {
|
||||
return [
|
||||
item('New Hampshire Department of Justice', 15, 580),
|
||||
item('Registered Charities List', 450, 581),
|
||||
item('Charitable Trusts Unit', 896, 580),
|
||||
item(
|
||||
'G = Good Standing; X = Not in Good Standing; S = Suspended',
|
||||
365,
|
||||
569,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
describe('reconstructRegistryRows', () => {
|
||||
it('returns no rows for an empty page', () => {
|
||||
expect(reconstructRegistryRows([[]])).toEqual([]);
|
||||
expect(reconstructRegistryRows([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('infers column boundaries from the header row and buckets data by x', () => {
|
||||
it('parses a full page: letterhead, legend, header, data rows', () => {
|
||||
const page = [
|
||||
...headerRow(900),
|
||||
item('Acme Foundation', NAME_X, 880),
|
||||
item('Concord', CITY_X, 880),
|
||||
item('Good Standing', STATUS_X, 880),
|
||||
...letterheadAndLegend(),
|
||||
...headerLine(547),
|
||||
...dataLine(532, {
|
||||
regNo: '35309',
|
||||
name: '22ZERO Follow Me In',
|
||||
address: 'PO Box 23',
|
||||
city: 'Pulaski',
|
||||
state: 'TN',
|
||||
zip: '38478',
|
||||
status: 'G',
|
||||
reportDue: '5/15/2027',
|
||||
}),
|
||||
...dataLine(518, {
|
||||
regNo: '33454',
|
||||
name: '#HappyPeriod',
|
||||
address: '4911 Mountain View Drive',
|
||||
city: 'Palmdale',
|
||||
state: 'CA',
|
||||
zip: '93552',
|
||||
status: 'X',
|
||||
reportDue: '9/14/2026',
|
||||
}),
|
||||
item('Updated: July 08, 2026', 15, 60),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page]);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('infers boundaries correctly even when they differ from other fixtures', () => {
|
||||
const page = [
|
||||
item('Organization', 40, 900),
|
||||
item('City', 250, 900),
|
||||
item('Status', 500, 900),
|
||||
// Slightly right of each boundary — still buckets into the same column.
|
||||
item('Bright Futures NH', 42, 870),
|
||||
item('Nashua', 255, 870),
|
||||
item('Active', 505, 870),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page]);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ name: 'Bright Futures NH', city: 'Nashua', status: 'Active' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('folds a multi-line org name (continuation line has empty city/status) into the previous row', () => {
|
||||
const page = [
|
||||
...headerRow(900),
|
||||
item('Very Long Nonprofit', NAME_X, 860),
|
||||
item('Manchester', CITY_X, 860),
|
||||
item('Lapsed', STATUS_X, 860),
|
||||
// Wrapped second line of the same org name — no city/status text.
|
||||
item('Name Incorporated', NAME_X, 840),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page]);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
name: 'Very Long Nonprofit Name Incorporated',
|
||||
city: 'Manchester',
|
||||
status: 'Lapsed',
|
||||
registrationNumber: '35309',
|
||||
name: '22ZERO Follow Me In',
|
||||
address: 'PO Box 23',
|
||||
city: 'Pulaski',
|
||||
state: 'TN',
|
||||
zip: '38478',
|
||||
status: 'G',
|
||||
reportDue: '5/15/2027',
|
||||
},
|
||||
{
|
||||
registrationNumber: '33454',
|
||||
name: '#HappyPeriod',
|
||||
address: '4911 Mountain View Drive',
|
||||
city: 'Palmdale',
|
||||
state: 'CA',
|
||||
zip: '93552',
|
||||
status: 'X',
|
||||
reportDue: '9/14/2026',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('supports multi-word continuation lines split across several text items', () => {
|
||||
it('folds a wrapped name line into the previous row', () => {
|
||||
const page = [
|
||||
...headerRow(900),
|
||||
item('Friends of the', NAME_X, 860),
|
||||
item('Merrimack', CITY_X, 860),
|
||||
item('Current', STATUS_X, 860),
|
||||
item('River', NAME_X, 840),
|
||||
item('Watershed', NAME_X + 40, 840),
|
||||
...headerLine(547),
|
||||
...dataLine(532, {
|
||||
regNo: '30456',
|
||||
name: '1st New Hampshire Light Battery',
|
||||
address: '11 Pinecrest Circle',
|
||||
city: 'Bedford',
|
||||
state: 'NH',
|
||||
zip: '03110',
|
||||
status: 'X',
|
||||
}),
|
||||
// Continuation: name column only — no reg no, no status.
|
||||
...dataLine(518, { name: 'Historical Association' }),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page]);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
name: 'Friends of the River Watershed',
|
||||
city: 'Merrimack',
|
||||
status: 'Current',
|
||||
},
|
||||
]);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]!.name).toBe(
|
||||
'1st New Hampshire Light Battery Historical Association',
|
||||
);
|
||||
});
|
||||
|
||||
it('drops header, letterhead, and page-number footer lines', () => {
|
||||
it('folds wrapped address text without disturbing the name', () => {
|
||||
const page = [
|
||||
item('State of New Hampshire Department of Justice', NAME_X, 950),
|
||||
...headerRow(900),
|
||||
item('Acme Foundation', NAME_X, 880),
|
||||
item('Concord', CITY_X, 880),
|
||||
item('Good Standing', STATUS_X, 880),
|
||||
item('3', NAME_X, 50),
|
||||
...headerLine(547),
|
||||
...dataLine(532, {
|
||||
regNo: '32030',
|
||||
name: '#WalkAway Foundation',
|
||||
address: '10521 Judicial Drive, Suite 200-A',
|
||||
city: 'Fairfax',
|
||||
state: 'VA',
|
||||
zip: '22030',
|
||||
status: 'G',
|
||||
}),
|
||||
...dataLine(518, { address: 'Fairfax, VA 22030' }),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page]);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' },
|
||||
]);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]!.name).toBe('#WalkAway Foundation');
|
||||
expect(rows[0]!.address).toBe(
|
||||
'10521 Judicial Drive, Suite 200-A Fairfax, VA 22030',
|
||||
);
|
||||
});
|
||||
|
||||
it('drops a "Page X of Y" footer line', () => {
|
||||
const page = [
|
||||
...headerRow(900),
|
||||
item('Acme Foundation', NAME_X, 880),
|
||||
item('Concord', CITY_X, 880),
|
||||
item('Good Standing', STATUS_X, 880),
|
||||
item('Page 1 of 12', NAME_X, 50),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page]);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('accepts explicit columnBoundaries for a continuation page with no repeated header', () => {
|
||||
const page = [
|
||||
item('Second Page Org', NAME_X, 900),
|
||||
item('Keene', CITY_X, 900),
|
||||
item('Unknown', STATUS_X, 900),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page], {
|
||||
columnBoundaries: [NAME_X, CITY_X, STATUS_X],
|
||||
});
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ name: 'Second Page Org', city: 'Keene', status: 'Unknown' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('reuses the last inferred boundaries across pages whose header does not repeat', () => {
|
||||
it('reuses the last-known anchors on header-less continuation pages', () => {
|
||||
const page1 = [
|
||||
...headerRow(900),
|
||||
item('First Page Org', NAME_X, 880),
|
||||
item('Concord', CITY_X, 880),
|
||||
item('Good Standing', STATUS_X, 880),
|
||||
...headerLine(547),
|
||||
...dataLine(532, {
|
||||
regNo: '1',
|
||||
name: 'Alpha',
|
||||
city: 'Concord',
|
||||
state: 'NH',
|
||||
status: 'G',
|
||||
}),
|
||||
];
|
||||
const page2 = [
|
||||
item('Second Page Org', NAME_X, 900),
|
||||
item('Keene', CITY_X, 900),
|
||||
item('Lapsed', STATUS_X, 900),
|
||||
...dataLine(532, {
|
||||
regNo: '2',
|
||||
name: 'Beta',
|
||||
city: 'Nashua',
|
||||
state: 'NH',
|
||||
status: 'S',
|
||||
}),
|
||||
];
|
||||
|
||||
const rows = reconstructRegistryRows([page1, page2]);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ name: 'First Page Org', city: 'Concord', status: 'Good Standing' },
|
||||
{ name: 'Second Page Org', city: 'Keene', status: 'Lapsed' },
|
||||
]);
|
||||
expect(rows.map((r) => r.name)).toEqual(['Alpha', 'Beta']);
|
||||
});
|
||||
|
||||
it('throws when a page has data but no boundaries can be determined', () => {
|
||||
const page = [item('Mystery Org', NAME_X, 900)];
|
||||
it('throws loudly when no anchors can be determined', () => {
|
||||
const page = [
|
||||
...dataLine(532, { regNo: '1', name: 'Alpha', status: 'G' }),
|
||||
];
|
||||
expect(() => reconstructRegistryRows([page])).toThrow(/column anchors/);
|
||||
});
|
||||
|
||||
expect(() => reconstructRegistryRows([page])).toThrow(
|
||||
/could not determine column boundaries/,
|
||||
);
|
||||
it('accepts explicit anchors and skips empty pages', () => {
|
||||
const anchors = [
|
||||
X.regNo,
|
||||
X.name,
|
||||
X.address,
|
||||
X.city,
|
||||
X.state,
|
||||
X.zip,
|
||||
X.status,
|
||||
X.reportDue,
|
||||
] as const;
|
||||
const page = [
|
||||
...dataLine(532, { regNo: '9', name: 'Gamma', status: 'G' }),
|
||||
];
|
||||
const rows = reconstructRegistryRows([[], page], {
|
||||
columnAnchors: anchors,
|
||||
});
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
registrationNumber: '9',
|
||||
name: 'Gamma',
|
||||
address: null,
|
||||
city: null,
|
||||
state: null,
|
||||
zip: null,
|
||||
status: 'G',
|
||||
reportDue: null,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeRegistrationStatus', () => {
|
||||
it.each([
|
||||
['Good Standing', 'good_standing'],
|
||||
['GOOD STANDING', 'good_standing'],
|
||||
['Current', 'good_standing'],
|
||||
['Active', 'good_standing'],
|
||||
['Lapsed', 'lapsed'],
|
||||
['Delinquent', 'lapsed'],
|
||||
['Suspended', 'lapsed'],
|
||||
['Expired', 'lapsed'],
|
||||
['Revoked', 'lapsed'],
|
||||
['', 'unknown'],
|
||||
['Pending Review', 'unknown'],
|
||||
] as const)('maps %s -> %s', (input, expected) => {
|
||||
expect(normalizeRegistrationStatus(input)).toBe(expected);
|
||||
it('maps the registry letter codes', () => {
|
||||
expect(normalizeRegistrationStatus('G')).toBe('good_standing');
|
||||
expect(normalizeRegistrationStatus('g')).toBe('good_standing');
|
||||
expect(normalizeRegistrationStatus('X')).toBe('lapsed');
|
||||
expect(normalizeRegistrationStatus('S')).toBe('suspended');
|
||||
});
|
||||
|
||||
it('maps spelled-out fallbacks, including the negated phrase', () => {
|
||||
expect(normalizeRegistrationStatus('Good Standing')).toBe('good_standing');
|
||||
expect(normalizeRegistrationStatus('Not in Good Standing')).toBe('lapsed');
|
||||
expect(normalizeRegistrationStatus('Suspended')).toBe('suspended');
|
||||
expect(normalizeRegistrationStatus('Revoked')).toBe('lapsed');
|
||||
});
|
||||
|
||||
it('maps blanks and surprises to unknown', () => {
|
||||
expect(normalizeRegistrationStatus('')).toBe('unknown');
|
||||
expect(normalizeRegistrationStatus(' ')).toBe('unknown');
|
||||
expect(normalizeRegistrationStatus('Q')).toBe('unknown');
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeRegistryRows', () => {
|
||||
it('maps status variants, trims/collapses whitespace, and preserves a null city', () => {
|
||||
const rows: RegistryRow[] = [
|
||||
{ name: ' Acme Foundation ', city: ' Concord ', status: 'Good Standing' },
|
||||
{ name: 'Delinquent Org', city: null, status: 'Delinquent' },
|
||||
{ name: 'Mystery Org', city: 'Keene', status: 'Something Else' },
|
||||
];
|
||||
|
||||
expect(normalizeRegistryRows(rows)).toEqual([
|
||||
{ name: 'Acme Foundation', city: 'Concord', status: 'good_standing' },
|
||||
{ name: 'Delinquent Org', city: null, status: 'lapsed' },
|
||||
{ name: 'Mystery Org', city: 'Keene', status: 'unknown' },
|
||||
it('trims to org fields, uppercases state, drops artifacts', () => {
|
||||
const rows = normalizeRegistryRows([
|
||||
{
|
||||
registrationNumber: '35309',
|
||||
name: ' 22ZERO Follow Me In ',
|
||||
address: 'PO Box 23',
|
||||
city: ' Pulaski ',
|
||||
state: 'tn',
|
||||
zip: '38478',
|
||||
status: 'G',
|
||||
reportDue: '5/15/2027',
|
||||
},
|
||||
{
|
||||
registrationNumber: null,
|
||||
name: 'Updated: July 08, 2026',
|
||||
address: null,
|
||||
city: null,
|
||||
state: null,
|
||||
zip: null,
|
||||
status: '',
|
||||
reportDue: null,
|
||||
},
|
||||
{
|
||||
registrationNumber: null,
|
||||
name: '',
|
||||
address: null,
|
||||
city: null,
|
||||
state: null,
|
||||
zip: null,
|
||||
status: 'G',
|
||||
reportDue: null,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops rows with an empty name and header/footer artifact rows that slipped through', () => {
|
||||
const rows: RegistryRow[] = [
|
||||
{ name: '', city: 'Concord', status: 'Good Standing' },
|
||||
{ name: 'Organization City Status', city: null, status: '' },
|
||||
{ name: 'Real Org', city: 'Nashua', status: 'Active' },
|
||||
];
|
||||
|
||||
expect(normalizeRegistryRows(rows)).toEqual([
|
||||
{ name: 'Real Org', city: 'Nashua', status: 'good_standing' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('collapses an empty-string city to null', () => {
|
||||
const rows: RegistryRow[] = [{ name: 'Org', city: ' ', status: 'Active' }];
|
||||
|
||||
expect(normalizeRegistryRows(rows)).toEqual([
|
||||
{ name: 'Org', city: null, status: 'good_standing' },
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
registrationNumber: '35309',
|
||||
name: '22ZERO Follow Me In',
|
||||
city: 'Pulaski',
|
||||
state: 'TN',
|
||||
status: 'good_standing',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,55 +1,92 @@
|
||||
/**
|
||||
* PURE row reconstruction over positioned text extracted from the NHDOJ
|
||||
* Charitable Trusts registry PDF (see `extract-pdf-text.ts` for the impure
|
||||
* layer that produces the input). No PDF or network dependency here — every
|
||||
* function operates on plain `PositionedTextItem[][]` so tests can drive it
|
||||
* with synthetic fixtures.
|
||||
* layer). No PDF or network dependency here — every function operates on
|
||||
* plain `PositionedTextItem[][]` so tests drive it with synthetic fixtures.
|
||||
*
|
||||
* The registry renders as a 3-column table (organization name, city,
|
||||
* registration status). pdf.js gives us a flat bag of positioned glyph runs
|
||||
* per page with no row/column structure, so reconstruction happens in two
|
||||
* passes:
|
||||
* Layout (verified against the real PDF, "Registered Charities List",
|
||||
* updated 2026-07-08, 427 pages): an 8-column table —
|
||||
*
|
||||
* Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due
|
||||
*
|
||||
* with a legend line "G = Good Standing; X = Not in Good Standing;
|
||||
* S = Suspended" under the letterhead, a repeated header row per page, and
|
||||
* an "Updated: <date>" footer. Status is a single letter (G/X/S). The list
|
||||
* includes out-of-state charities registered to solicit in NH, so State is
|
||||
* carried through rather than assumed 'NH'.
|
||||
*
|
||||
* Reconstruction:
|
||||
* 1. `reconstructRegistryRows` groups items into visual lines by y
|
||||
* (tolerance ~2pt), buckets each line's items into columns by x
|
||||
* (boundaries inferred from the header row's token x-positions, or
|
||||
* supplied explicitly via `options.columnBoundaries` when a page's
|
||||
* header doesn't repeat), and stitches multi-line org names back
|
||||
* together — a continuation line has text only in the name column.
|
||||
* 2. `normalizeRegistryRows` maps the free-text status column onto the
|
||||
* tri-state the DB expects and drops anything that isn't really a data
|
||||
* row (repeated header, page number, agency letterhead).
|
||||
* (~2pt tolerance), buckets each line's items into the 8 columns by x
|
||||
* (anchors inferred from the header row's token positions, or supplied
|
||||
* via `options.columnAnchors`), and folds wrapped lines — a line with
|
||||
* no Reg. No. and no Status continues the previous row's name/address.
|
||||
* 2. `normalizeRegistryRows` maps G/X/S onto the status enum and drops
|
||||
* artifact rows (legend, letterhead, page footers).
|
||||
*/
|
||||
import type { PositionedTextItem } from './extract-pdf-text.js';
|
||||
|
||||
export interface RegistryRow {
|
||||
readonly registrationNumber: string | null;
|
||||
readonly name: string;
|
||||
readonly address: string | null;
|
||||
readonly city: string | null;
|
||||
readonly state: string | null;
|
||||
readonly zip: string | null;
|
||||
readonly status: string;
|
||||
readonly reportDue: string | null;
|
||||
}
|
||||
|
||||
export type NormalizedRegistrationStatus = 'good_standing' | 'lapsed' | 'unknown';
|
||||
export type NormalizedRegistrationStatus =
|
||||
| 'good_standing'
|
||||
| 'lapsed'
|
||||
| 'suspended'
|
||||
| 'unknown';
|
||||
|
||||
export interface NormalizedRegistryRow {
|
||||
readonly registrationNumber: string | null;
|
||||
readonly name: string;
|
||||
readonly city: string | null;
|
||||
readonly state: string | null;
|
||||
readonly status: NormalizedRegistrationStatus;
|
||||
}
|
||||
|
||||
/** Ascending x-anchors for the 8 columns, left edge of each. */
|
||||
export type ColumnAnchors = readonly [
|
||||
number, // Reg. No.
|
||||
number, // Charity Name
|
||||
number, // Address
|
||||
number, // City
|
||||
number, // State
|
||||
number, // Zip
|
||||
number, // Status
|
||||
number, // Report Due
|
||||
];
|
||||
|
||||
export interface ParseRegistryOptions {
|
||||
/** Max y-distance (pt) between items considered part of the same visual line. Default 2. */
|
||||
/** Max y-distance (pt) between items on the same visual line. Default 2. */
|
||||
readonly yTolerance?: number;
|
||||
/**
|
||||
* Explicit x-position column starts `[nameStart, cityStart, statusStart]`,
|
||||
* ascending. When omitted, boundaries are inferred per page from that
|
||||
* page's header row (falling back to the most recently inferred/ provided
|
||||
* boundaries for pages whose header doesn't repeat, e.g. continuation
|
||||
* pages).
|
||||
* Explicit column anchors. When omitted, inferred per page from that
|
||||
* page's header row; pages without a header reuse the last-known anchors.
|
||||
*/
|
||||
readonly columnBoundaries?: readonly [number, number, number];
|
||||
readonly columnAnchors?: ColumnAnchors;
|
||||
}
|
||||
|
||||
const DEFAULT_Y_TOLERANCE = 2;
|
||||
/** Data cells sit within a few pt left of their header token's x. */
|
||||
const COLUMN_X_SLACK = 4;
|
||||
|
||||
const HEADER_PATTERNS: readonly RegExp[] = [
|
||||
/^reg\.?\s*no\.?$/i,
|
||||
/^charity\s+name$/i,
|
||||
/^address$/i,
|
||||
/^city$/i,
|
||||
/^state$/i,
|
||||
/^zip$/i,
|
||||
/^status$/i,
|
||||
/^report\s+due$/i,
|
||||
];
|
||||
|
||||
interface Line {
|
||||
readonly y: number;
|
||||
@@ -60,10 +97,7 @@ function collapseWhitespace(text: string): string {
|
||||
return text.trim().replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups a page's positioned items into visual lines (top-to-bottom by y,
|
||||
* items within a line ordered left-to-right by x).
|
||||
*/
|
||||
/** Groups items into visual lines: top-to-bottom, left-to-right. */
|
||||
function groupLines(
|
||||
items: readonly PositionedTextItem[],
|
||||
yTolerance: number,
|
||||
@@ -88,33 +122,29 @@ function groupLines(
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Bucket an x-position into a column index given ascending column-start boundaries. */
|
||||
function columnIndexForX(x: number, boundaries: readonly number[]): number {
|
||||
for (let i = boundaries.length - 1; i >= 0; i--) {
|
||||
if (x >= boundaries[i]!) return i;
|
||||
function columnIndexForX(x: number, anchors: ColumnAnchors): number {
|
||||
for (let i = anchors.length - 1; i >= 0; i--) {
|
||||
if (x >= anchors[i]! - COLUMN_X_SLACK) return i;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks for a line whose items spell out the three column headers
|
||||
* ("Organization"/"Name", "City", "Status") and returns their x-positions,
|
||||
* ascending — which is also left-to-right table order (name, city, status).
|
||||
* Finds the header row (all 8 column labels on one line) and returns each
|
||||
* label's x-position as the column anchors.
|
||||
*/
|
||||
function inferColumnBoundariesFromHeader(
|
||||
function inferColumnAnchorsFromHeader(
|
||||
lines: readonly Line[],
|
||||
): [number, number, number] | null {
|
||||
): ColumnAnchors | null {
|
||||
for (const line of lines) {
|
||||
const nameItem = line.items.find((i) =>
|
||||
/organi[sz]ation|^name$/i.test(i.str.trim()),
|
||||
);
|
||||
const cityItem = line.items.find((i) => /^city$/i.test(i.str.trim()));
|
||||
const statusItem = line.items.find((i) => /status/i.test(i.str.trim()));
|
||||
|
||||
if (nameItem != null && cityItem != null && statusItem != null) {
|
||||
return [nameItem.x, cityItem.x, statusItem.x].sort(
|
||||
(a, b) => a - b,
|
||||
) as [number, number, number];
|
||||
const anchors: number[] = [];
|
||||
for (const pattern of HEADER_PATTERNS) {
|
||||
const hit = line.items.find((i) => pattern.test(i.str.trim()));
|
||||
if (hit == null) break;
|
||||
anchors.push(hit.x);
|
||||
}
|
||||
if (anchors.length === HEADER_PATTERNS.length) {
|
||||
return [...anchors].sort((a, b) => a - b) as unknown as ColumnAnchors;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
@@ -123,19 +153,23 @@ function inferColumnBoundariesFromHeader(
|
||||
function isColumnHeaderLine(lineText: string): boolean {
|
||||
const upper = lineText.toUpperCase();
|
||||
return (
|
||||
(upper.includes('ORGANIZATION') || /\bNAME\b/.test(upper)) &&
|
||||
upper.includes('CHARITY NAME') &&
|
||||
upper.includes('CITY') &&
|
||||
upper.includes('STATUS')
|
||||
);
|
||||
}
|
||||
|
||||
/** Page-number footers, repeated agency letterhead, and blank lines. */
|
||||
/** Legend, letterhead, page footers, and blank lines. */
|
||||
function isFooterOrArtifactLine(lineText: string): boolean {
|
||||
const trimmed = lineText.trim();
|
||||
if (trimmed === '') return true;
|
||||
if (/^\d+$/.test(trimmed)) return true;
|
||||
if (/^page\s+\d+(\s+of\s+\d+)?$/i.test(trimmed)) return true;
|
||||
if (/department of justice/i.test(trimmed)) return true;
|
||||
if (/registered charities list/i.test(trimmed)) return true;
|
||||
if (/charitable trusts unit/i.test(trimmed)) return true;
|
||||
if (/=\s*good standing/i.test(trimmed)) return true;
|
||||
if (/^updated:/i.test(trimmed)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -144,22 +178,19 @@ function isHeaderOrFooterLine(lineText: string): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstructs `{ name, city, status }` rows from positioned text, one page
|
||||
* array at a time. Multi-line org names (a continuation line with nothing
|
||||
* in the city/status columns) are folded back into the previous row.
|
||||
* Reconstructs registry rows from positioned text, one page at a time.
|
||||
* Wrapped rows (no Reg. No., no Status) fold their name/address text back
|
||||
* into the previous row.
|
||||
*
|
||||
* Throws if a page has data but no column boundaries can be determined
|
||||
* (no header row found on any page so far, and none supplied) — that means
|
||||
* the registry's layout changed and silent misparsing is worse than a loud
|
||||
* failure here.
|
||||
* Throws if a page has data but no column anchors can be determined —
|
||||
* a layout change should fail loudly, not misparse silently.
|
||||
*/
|
||||
export function reconstructRegistryRows(
|
||||
pages: readonly (readonly PositionedTextItem[])[],
|
||||
options: ParseRegistryOptions = {},
|
||||
): RegistryRow[] {
|
||||
const yTolerance = options.yTolerance ?? DEFAULT_Y_TOLERANCE;
|
||||
let boundaries: readonly [number, number, number] | null =
|
||||
options.columnBoundaries ?? null;
|
||||
let anchors: ColumnAnchors | null = options.columnAnchors ?? null;
|
||||
|
||||
const rows: RegistryRow[] = [];
|
||||
|
||||
@@ -168,15 +199,15 @@ export function reconstructRegistryRows(
|
||||
|
||||
const lines = groupLines(pageItems, yTolerance);
|
||||
|
||||
if (options.columnBoundaries == null) {
|
||||
const inferred = inferColumnBoundariesFromHeader(lines);
|
||||
if (inferred != null) boundaries = inferred;
|
||||
if (options.columnAnchors == null) {
|
||||
const inferred = inferColumnAnchorsFromHeader(lines);
|
||||
if (inferred != null) anchors = inferred;
|
||||
}
|
||||
|
||||
if (boundaries == null) {
|
||||
if (anchors == null) {
|
||||
throw new Error(
|
||||
'reconstructRegistryRows: could not determine column boundaries ' +
|
||||
'(no header row found and none provided via options.columnBoundaries)',
|
||||
'reconstructRegistryRows: could not determine column anchors ' +
|
||||
'(no header row found and none provided via options.columnAnchors)',
|
||||
);
|
||||
}
|
||||
|
||||
@@ -184,60 +215,97 @@ export function reconstructRegistryRows(
|
||||
const lineText = line.items.map((i) => i.str).join(' ');
|
||||
if (isHeaderOrFooterLine(lineText)) continue;
|
||||
|
||||
const columns: [string[], string[], string[]] = [[], [], []];
|
||||
const columns: string[][] = Array.from(
|
||||
{ length: HEADER_PATTERNS.length },
|
||||
() => [],
|
||||
);
|
||||
for (const item of line.items) {
|
||||
const idx = columnIndexForX(item.x, boundaries);
|
||||
columns[idx as 0 | 1 | 2].push(item.str);
|
||||
if (item.str.trim() === '') continue;
|
||||
columns[columnIndexForX(item.x, anchors)]!.push(item.str);
|
||||
}
|
||||
|
||||
const name = collapseWhitespace(columns[0].join(' '));
|
||||
const city = collapseWhitespace(columns[1].join(' '));
|
||||
const status = collapseWhitespace(columns[2].join(' '));
|
||||
const cell = (i: number): string =>
|
||||
collapseWhitespace(columns[i]!.join(' '));
|
||||
|
||||
if (name === '' && city === '' && status === '') continue;
|
||||
const registrationNumber = cell(0);
|
||||
const name = cell(1);
|
||||
const address = cell(2);
|
||||
const city = cell(3);
|
||||
const state = cell(4);
|
||||
const zip = cell(5);
|
||||
const status = cell(6);
|
||||
const reportDue = cell(7);
|
||||
|
||||
// A line with text only in the name column continues the previous
|
||||
// row's (wrapped) org name rather than starting a new row.
|
||||
if (city === '' && status === '' && rows.length > 0) {
|
||||
if (
|
||||
[registrationNumber, name, address, city, state, zip, status, reportDue]
|
||||
.every((c) => c === '')
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// A wrapped row: no registry number and no status — its name/address
|
||||
// text continues the previous row rather than starting a new one.
|
||||
if (registrationNumber === '' && status === '' && rows.length > 0) {
|
||||
const prev = rows[rows.length - 1]!;
|
||||
rows[rows.length - 1] = {
|
||||
...prev,
|
||||
name: collapseWhitespace(`${prev.name} ${name}`),
|
||||
address:
|
||||
address === ''
|
||||
? prev.address
|
||||
: collapseWhitespace(`${prev.address ?? ''} ${address}`),
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.push({ name, city: city === '' ? null : city, status });
|
||||
rows.push({
|
||||
registrationNumber: registrationNumber === '' ? null : registrationNumber,
|
||||
name,
|
||||
address: address === '' ? null : address,
|
||||
city: city === '' ? null : city,
|
||||
state: state === '' ? null : state,
|
||||
zip: zip === '' ? null : zip,
|
||||
status,
|
||||
reportDue: reportDue === '' ? null : reportDue,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
const GOOD_STANDING_PATTERNS = [/good\s*standing/i, /\bcurrent\b/i, /\bactive\b/i];
|
||||
const LAPSED_PATTERNS = [
|
||||
/\blapsed\b/i,
|
||||
/\bdelinquent\b/i,
|
||||
/\bsuspended\b/i,
|
||||
/\bexpired\b/i,
|
||||
/\brevoked\b/i,
|
||||
];
|
||||
|
||||
/**
|
||||
* Registry legend: G = Good Standing; X = Not in Good Standing;
|
||||
* S = Suspended. Phrase forms accepted as a fallback for older
|
||||
* republishes that spelled statuses out.
|
||||
*/
|
||||
export function normalizeRegistrationStatus(
|
||||
status: string,
|
||||
): NormalizedRegistrationStatus {
|
||||
const trimmed = status.trim();
|
||||
if (trimmed === '') return 'unknown';
|
||||
if (GOOD_STANDING_PATTERNS.some((p) => p.test(trimmed))) return 'good_standing';
|
||||
if (LAPSED_PATTERNS.some((p) => p.test(trimmed))) return 'lapsed';
|
||||
if (/^g$/i.test(trimmed) || /good\s*standing/i.test(trimmed)) {
|
||||
// "Not in Good Standing" also contains the phrase — X handles the real
|
||||
// file; this phrase fallback must not swallow the negated form.
|
||||
if (/not\s+in\s+good/i.test(trimmed)) return 'lapsed';
|
||||
return 'good_standing';
|
||||
}
|
||||
if (/^s$/i.test(trimmed) || /\bsuspended\b/i.test(trimmed)) return 'suspended';
|
||||
if (
|
||||
/^x$/i.test(trimmed) ||
|
||||
/\blapsed\b/i.test(trimmed) ||
|
||||
/\bdelinquent\b/i.test(trimmed) ||
|
||||
/\bexpired\b/i.test(trimmed) ||
|
||||
/\brevoked\b/i.test(trimmed)
|
||||
) {
|
||||
return 'lapsed';
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps raw rows onto the tri-state status the DB expects, collapses
|
||||
* whitespace, and drops anything that isn't really a registrant (a
|
||||
* repeated header row or letterhead line that slipped through row
|
||||
* reconstruction as a degenerate one-column row).
|
||||
* Maps raw rows onto the DB's status enum, trims to the fields the org
|
||||
* table stores, and drops anything that isn't really a registrant.
|
||||
*/
|
||||
export function normalizeRegistryRows(
|
||||
rows: readonly RegistryRow[],
|
||||
@@ -249,9 +317,14 @@ export function normalizeRegistryRows(
|
||||
if (name === '' || isHeaderOrFooterLine(name)) continue;
|
||||
|
||||
const city = row.city == null ? null : collapseWhitespace(row.city);
|
||||
const state =
|
||||
row.state == null ? null : collapseWhitespace(row.state).toUpperCase();
|
||||
|
||||
normalized.push({
|
||||
registrationNumber: row.registrationNumber,
|
||||
name,
|
||||
city: city === '' ? null : city,
|
||||
state: state === '' ? null : state,
|
||||
status: normalizeRegistrationStatus(row.status),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -62,6 +62,13 @@ export async function searchOrganizations(
|
||||
url.searchParams.set('state[id]', state);
|
||||
|
||||
const res = await fetch(url);
|
||||
if (res.status === 404) {
|
||||
// Nonprofit Explorer quirk (verified 2026-07-16): a filtered search
|
||||
// with zero hits responds 404, not 200-with-empty-list. Zero hits is a
|
||||
// legitimate "no match" — the org resolves as unresolved, not failed.
|
||||
await wait(options.politenessDelayMs ?? DEFAULT_POLITENESS_DELAY_MS);
|
||||
return { organizations: [] };
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`ProPublica search failed for "${name}" (${state}): ${res.status} ${res.statusText}`,
|
||||
|
||||
@@ -181,6 +181,7 @@ async function runEnrichOrgs(): Promise<void> {
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachEnrichOrgsRegistered?: boolean;
|
||||
__outreachEnrichOrgsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
|
||||
};
|
||||
|
||||
if (!g.__outreachEnrichOrgsRegistered) {
|
||||
@@ -197,10 +198,26 @@ if (!g.__outreachEnrichOrgsRegistered) {
|
||||
|
||||
// Must be registered as BOTH a workflow and a scheduled function,
|
||||
// referencing the same function object — see module doc comment.
|
||||
DBOS.registerWorkflow(enrichOrgs, { name: 'enrichOrgs' });
|
||||
g.__outreachEnrichOrgsHandle = DBOS.registerWorkflow(enrichOrgs, {
|
||||
name: 'enrichOrgs',
|
||||
});
|
||||
DBOS.registerScheduled(enrichOrgs, {
|
||||
crontab: '0 5 * * *',
|
||||
name: 'enrichOrgs',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts one durable run of this workflow immediately through DBOS —
|
||||
* the exact production path (workflow + checkpointed steps), used by
|
||||
* `run-once.ts` for supervised/manual passes. Requires deps injected and
|
||||
* `DBOS.launch()` completed.
|
||||
*/
|
||||
export function runEnrichOrgsNow(): Promise<void> {
|
||||
const handle = g.__outreachEnrichOrgsHandle;
|
||||
if (handle == null) {
|
||||
throw new Error('enrichOrgs is not registered; was this module imported before DBOS.launch()?');
|
||||
}
|
||||
return handle(new Date(), new Date());
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@ async function runExpireGrants(): Promise<void> {
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachExpireGrantsRegistered?: boolean;
|
||||
__outreachExpireGrantsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
|
||||
};
|
||||
|
||||
if (!g.__outreachExpireGrantsRegistered) {
|
||||
@@ -61,10 +62,26 @@ if (!g.__outreachExpireGrantsRegistered) {
|
||||
}
|
||||
};
|
||||
|
||||
DBOS.registerWorkflow(expireGrants, { name: 'expireGrants' });
|
||||
g.__outreachExpireGrantsHandle = DBOS.registerWorkflow(expireGrants, {
|
||||
name: 'expireGrants',
|
||||
});
|
||||
DBOS.registerScheduled(expireGrants, {
|
||||
crontab: '0 * * * *',
|
||||
name: 'expireGrants',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts one durable run of this workflow immediately through DBOS —
|
||||
* the exact production path (workflow + checkpointed steps), used by
|
||||
* `run-once.ts` for supervised/manual passes. Requires deps injected and
|
||||
* `DBOS.launch()` completed.
|
||||
*/
|
||||
export function runExpireGrantsNow(): Promise<void> {
|
||||
const handle = g.__outreachExpireGrantsHandle;
|
||||
if (handle == null) {
|
||||
throw new Error('expireGrants is not registered; was this module imported before DBOS.launch()?');
|
||||
}
|
||||
return handle(new Date(), new Date());
|
||||
}
|
||||
|
||||
@@ -154,6 +154,7 @@ async function runIngestGrants(): Promise<void> {
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachIngestGrantsRegistered?: boolean;
|
||||
__outreachIngestGrantsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
|
||||
};
|
||||
|
||||
if (!g.__outreachIngestGrantsRegistered) {
|
||||
@@ -170,10 +171,26 @@ if (!g.__outreachIngestGrantsRegistered) {
|
||||
|
||||
// Must be registered as BOTH a workflow and a scheduled function,
|
||||
// referencing the same function object — see module doc comment.
|
||||
DBOS.registerWorkflow(ingestGrants, { name: 'ingestGrants' });
|
||||
g.__outreachIngestGrantsHandle = DBOS.registerWorkflow(ingestGrants, {
|
||||
name: 'ingestGrants',
|
||||
});
|
||||
DBOS.registerScheduled(ingestGrants, {
|
||||
crontab: '0 3 * * *',
|
||||
name: 'ingestGrants',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts one durable run of this workflow immediately through DBOS —
|
||||
* the exact production path (workflow + checkpointed steps), used by
|
||||
* `run-once.ts` for supervised/manual passes. Requires deps injected and
|
||||
* `DBOS.launch()` completed.
|
||||
*/
|
||||
export function runIngestGrantsNow(): Promise<void> {
|
||||
const handle = g.__outreachIngestGrantsHandle;
|
||||
if (handle == null) {
|
||||
throw new Error('ingestGrants is not registered; was this module imported before DBOS.launch()?');
|
||||
}
|
||||
return handle(new Date(), new Date());
|
||||
}
|
||||
|
||||
@@ -50,12 +50,29 @@ function getIngestNhdojOrgsDeps(): IngestNhdojOrgsDeps {
|
||||
}
|
||||
|
||||
async function fetchNhdojRegistryPdf(url: string): Promise<Uint8Array> {
|
||||
const res = await fetch(url);
|
||||
// mm.nh.gov sits behind Akamai bot detection: bare curl/default clients
|
||||
// get a 403 "Access Denied" HTML page. Node's fetch passes with
|
||||
// browser-like headers (verified 2026-07-16).
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
||||
Accept: 'application/pdf,*/*',
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`fetchNhdojRegistryPdf: ${url} responded ${res.status} ${res.statusText}`,
|
||||
);
|
||||
}
|
||||
const contentType = res.headers.get('content-type') ?? '';
|
||||
if (!contentType.includes('pdf')) {
|
||||
// Akamai serves its denial as 200 text/html through some paths — treat
|
||||
// a non-PDF body as a failure rather than feeding HTML to pdfjs.
|
||||
throw new Error(
|
||||
`fetchNhdojRegistryPdf: expected a PDF but got content-type "${contentType}" from ${url}`,
|
||||
);
|
||||
}
|
||||
const buf = await res.arrayBuffer();
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
@@ -72,6 +89,10 @@ async function upsertRegistryOrg(
|
||||
return serverUpsertOrgFromRegistry(db, {
|
||||
name: row.name,
|
||||
city: row.city,
|
||||
// The registry includes out-of-state charities registered to solicit
|
||||
// in NH — carry their real state so the geography gate stays honest.
|
||||
state: row.state,
|
||||
registrationNumber: row.registrationNumber,
|
||||
registrationStatus: row.status,
|
||||
sourceRegistry: SOURCE_REGISTRY,
|
||||
});
|
||||
@@ -85,18 +106,25 @@ const upsertRegistryOrgStep = DBOS.registerStep(upsertRegistryOrg, {
|
||||
async function runIngestNhdojOrgs(): Promise<void> {
|
||||
const { db } = getIngestNhdojOrgsDeps();
|
||||
|
||||
const pdfPath = process.env.NHDOJ_REGISTRY_PDF_PATH;
|
||||
const pdfUrl = process.env.NHDOJ_REGISTRY_PDF_URL;
|
||||
if (pdfUrl == null || pdfUrl.trim() === '') {
|
||||
|
||||
let pdfBytes: Uint8Array;
|
||||
if (pdfPath != null && pdfPath.trim() !== '') {
|
||||
// Local-file escape hatch for supervised runs and Akamai outages.
|
||||
const { readFile } = await import('node:fs/promises');
|
||||
pdfBytes = new Uint8Array(await readFile(pdfPath));
|
||||
} else if (pdfUrl != null && pdfUrl.trim() !== '') {
|
||||
pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl);
|
||||
} else {
|
||||
// The registry PDF's URL changes whenever NHDOJ republishes it (no
|
||||
// stable endpoint) — a hard failure here would page on-call for a
|
||||
// config gap rather than a real outage, so skip quietly instead.
|
||||
console.warn(
|
||||
'[ingest-nhdoj-orgs] NHDOJ_REGISTRY_PDF_URL is not set; skipping this pass.',
|
||||
'[ingest-nhdoj-orgs] neither NHDOJ_REGISTRY_PDF_PATH nor NHDOJ_REGISTRY_PDF_URL is set; skipping this pass.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl);
|
||||
const pages = await extractPositionedText(pdfBytes);
|
||||
const rawRows = reconstructRegistryRows(pages);
|
||||
const normalizedRows = normalizeRegistryRows(rawRows);
|
||||
@@ -116,6 +144,7 @@ async function runIngestNhdojOrgs(): Promise<void> {
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachIngestNhdojOrgsRegistered?: boolean;
|
||||
__outreachIngestNhdojOrgsHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
|
||||
};
|
||||
|
||||
if (!g.__outreachIngestNhdojOrgsRegistered) {
|
||||
@@ -132,10 +161,26 @@ if (!g.__outreachIngestNhdojOrgsRegistered) {
|
||||
|
||||
// Must be registered as BOTH a workflow and a scheduled function,
|
||||
// referencing the same function object — see module doc comment.
|
||||
DBOS.registerWorkflow(ingestNhdojOrgs, { name: 'ingestNhdojOrgs' });
|
||||
g.__outreachIngestNhdojOrgsHandle = DBOS.registerWorkflow(ingestNhdojOrgs, {
|
||||
name: 'ingestNhdojOrgs',
|
||||
});
|
||||
DBOS.registerScheduled(ingestNhdojOrgs, {
|
||||
crontab: '0 4 1 * *',
|
||||
name: 'ingestNhdojOrgs',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts one durable run of this workflow immediately through DBOS —
|
||||
* the exact production path (workflow + checkpointed steps), used by
|
||||
* `run-once.ts` for supervised/manual passes. Requires deps injected and
|
||||
* `DBOS.launch()` completed.
|
||||
*/
|
||||
export function runIngestNhdojOrgsNow(): Promise<void> {
|
||||
const handle = g.__outreachIngestNhdojOrgsHandle;
|
||||
if (handle == null) {
|
||||
throw new Error('ingestNhdojOrgs is not registered; was this module imported before DBOS.launch()?');
|
||||
}
|
||||
return handle(new Date(), new Date());
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ async function runIngestPndRss(): Promise<void> {
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachIngestPndRssRegistered?: boolean;
|
||||
__outreachIngestPndRssHandle?: (scheduledTime: Date, startedAt: Date) => Promise<void>;
|
||||
};
|
||||
|
||||
if (!g.__outreachIngestPndRssRegistered) {
|
||||
@@ -104,10 +105,26 @@ if (!g.__outreachIngestPndRssRegistered) {
|
||||
|
||||
// Must be registered as BOTH a workflow and a scheduled function,
|
||||
// referencing the same function object — see module doc comment.
|
||||
DBOS.registerWorkflow(ingestPndRss, { name: 'ingestPndRss' });
|
||||
g.__outreachIngestPndRssHandle = DBOS.registerWorkflow(ingestPndRss, {
|
||||
name: 'ingestPndRss',
|
||||
});
|
||||
DBOS.registerScheduled(ingestPndRss, {
|
||||
crontab: '30 3 * * *',
|
||||
name: 'ingestPndRss',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts one durable run of this workflow immediately through DBOS —
|
||||
* the exact production path (workflow + checkpointed steps), used by
|
||||
* `run-once.ts` for supervised/manual passes. Requires deps injected and
|
||||
* `DBOS.launch()` completed.
|
||||
*/
|
||||
export function runIngestPndRssNow(): Promise<void> {
|
||||
const handle = g.__outreachIngestPndRssHandle;
|
||||
if (handle == null) {
|
||||
throw new Error('ingestPndRss is not registered; was this module imported before DBOS.launch()?');
|
||||
}
|
||||
return handle(new Date(), new Date());
|
||||
}
|
||||
|
||||
@@ -47,6 +47,13 @@ Grant upserts key on `grants.source_url`; org registry upserts key on case-insen
|
||||
|
||||
### Philanthropy News Digest RSS
|
||||
|
||||
> **Status (2026-07-16): feed retired upstream.** Every historical feed path
|
||||
> (`/feeds/rfps.rss` and variants) now returns the site's HTML shell — PND is
|
||||
> a Next.js app with client-side data and no `<link rel=alternate>` feeds.
|
||||
> The workflow runs and upserts zero rows. Rework candidate: drive their
|
||||
> internal JSON API / `__NEXT_DATA__` instead. Low priority — PND was "cheap
|
||||
> incremental coverage"; Grants.gov + NH state sources carry the corpus.
|
||||
|
||||
Nightly ingestion of the Philanthropy News Digest "RFPs" RSS feed (`https://philanthropynewsdigest.org/feeds/rfps.rss`, overridable via `PND_RFP_FEED_URL`).
|
||||
|
||||
- **Client** — `src/sources/pnd-rss/client.ts`: `fetchPndRfpFeed(feedUrl)` does a plain `fetch`, throwing on any non-2xx response.
|
||||
@@ -94,3 +101,10 @@ Monthly re-scan of the NH Department of Justice Charitable Trusts Unit's registr
|
||||
// ...
|
||||
setIngestNhdojOrgsDeps({ db });
|
||||
```
|
||||
|
||||
## First-run field findings (2026-07-16)
|
||||
|
||||
- **NHDOJ registry**: 8-column layout (`Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due`), single-letter statuses (G/X/S legend), includes out-of-state charities registered to solicit in NH. Parsed 13,709 registrants → 13,632 orgs (6,266 NH). `Reg. No.` is the stable upsert key (`orgs.registration_number`). mm.nh.gov sits behind Akamai: bare curl gets 403; Node fetch with browser-like headers passes (client sends them). `NHDOJ_REGISTRY_PDF_PATH` overrides the URL for supervised runs.
|
||||
- **ProPublica**: a state-scoped search with zero hits returns **404**, not an empty list — the client maps 404 → no candidates (org resolves `unresolved`). Before the fix this tripped the 20% systemic-failure breaker.
|
||||
- **Grants.gov**: first pass ingested 200 opportunities (search cap 1000 / detail cap 200); the hourly expiry sweep correctly expired a same-day deadline immediately.
|
||||
- **Enrichment queue** prioritizes NH + good-standing orgs before the out-of-state tail; ~60% of NH orgs resolve an EIN per batch, the rest record an all-null attempt.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TYPE "public"."registration_status" ADD VALUE IF NOT EXISTS 'suspended' BEFORE 'unknown';--> statement-breakpoint
|
||||
ALTER TABLE "orgs" ADD COLUMN "registration_number" text;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "idx_orgs_registry_reg_no" ON "orgs" USING btree ("source_registry","registration_number") WHERE "orgs"."source_registry" IS NOT NULL AND "orgs"."registration_number" IS NOT NULL;
|
||||
1149
packages/outreach-core/drizzle/server/meta/1784231856_snapshot.json
Normal file
1149
packages/outreach-core/drizzle/server/meta/1784231856_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,13 @@
|
||||
"when": 1784214387212,
|
||||
"tag": "1784214387_initial-schema",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "7",
|
||||
"when": 1784231856865,
|
||||
"tag": "1784231856_nhdoj-registry-fields",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -43,6 +43,7 @@ export const grantStatusEnum = pgEnum('grant_status', [
|
||||
export const registrationStatusEnum = pgEnum('registration_status', [
|
||||
'good_standing',
|
||||
'lapsed',
|
||||
'suspended',
|
||||
'unknown',
|
||||
]);
|
||||
|
||||
@@ -174,6 +175,9 @@ export const orgs = pgTable(
|
||||
nteeCode: text('ntee_code'),
|
||||
totalRevenue: integer('total_revenue'),
|
||||
fiscalYearEnd: text('fiscal_year_end'),
|
||||
// Registry-assigned identifier (e.g. NHDOJ "Reg. No.") — stable across
|
||||
// republishes, unlike org names; the preferred upsert key when present.
|
||||
registrationNumber: text('registration_number'),
|
||||
registrationStatus: registrationStatusEnum('registration_status')
|
||||
.notNull()
|
||||
.default('unknown'),
|
||||
@@ -186,6 +190,11 @@ export const orgs = pgTable(
|
||||
// Partial unique index: EIN is unique when present, but many small
|
||||
// orgs in early ingestion won't have one resolved yet.
|
||||
uniqueIndex('idx_orgs_ein').on(t.ein).where(sql`${t.ein} IS NOT NULL`),
|
||||
uniqueIndex('idx_orgs_registry_reg_no')
|
||||
.on(t.sourceRegistry, t.registrationNumber)
|
||||
.where(
|
||||
sql`${t.sourceRegistry} IS NOT NULL AND ${t.registrationNumber} IS NOT NULL`,
|
||||
),
|
||||
index('idx_orgs_icp_band').on(t.icpBand),
|
||||
index('idx_orgs_state').on(t.state),
|
||||
],
|
||||
|
||||
@@ -6,8 +6,10 @@ import { schema } from '#~/db/db.js';
|
||||
export interface RegistryOrgInput {
|
||||
readonly name: string;
|
||||
readonly city: string | null;
|
||||
readonly state?: string;
|
||||
readonly registrationStatus: 'good_standing' | 'lapsed' | 'unknown';
|
||||
readonly state?: string | null;
|
||||
/** Registry-assigned id (e.g. NHDOJ Reg. No.) — preferred identity when present. */
|
||||
readonly registrationNumber?: string | null;
|
||||
readonly registrationStatus: 'good_standing' | 'lapsed' | 'suspended' | 'unknown';
|
||||
readonly sourceRegistry: string;
|
||||
}
|
||||
|
||||
@@ -18,12 +20,17 @@ export interface RegistryUpsertResult {
|
||||
|
||||
/**
|
||||
* Upserts an org discovered in a charity registry re-scan (e.g. the NHDOJ
|
||||
* Charitable Trusts list). Registries carry no EIN, so identity is the
|
||||
* case-insensitive (name, city, state) triple — good enough for a
|
||||
* single-state registry where the same legal name in the same city is the
|
||||
* same org. Existing rows get their registration status refreshed (orgs
|
||||
* lapse and re-register); enrichment fields (EIN, revenue, NTEE) are never
|
||||
* touched here.
|
||||
* Charitable Trusts list).
|
||||
*
|
||||
* Identity, in preference order:
|
||||
* 1. (sourceRegistry, registrationNumber) — the registry's own stable id;
|
||||
* survives renames and relocations. Name/city/state are refreshed from
|
||||
* the incoming row on this path.
|
||||
* 2. Case-insensitive (name, city, state) — fallback for registries (or
|
||||
* historical rows) without a registration number. Only status and
|
||||
* provenance are refreshed on this path.
|
||||
*
|
||||
* Enrichment fields (EIN, revenue, NTEE) are never touched here.
|
||||
*
|
||||
* Returns `inserted: true` for brand-new registrants — a segment the
|
||||
* pipeline treats specially (often actively seeking first-time funding).
|
||||
@@ -33,8 +40,37 @@ export async function serverUpsertOrgFromRegistry(
|
||||
org: RegistryOrgInput,
|
||||
): Promise<RegistryUpsertResult> {
|
||||
const state = org.state ?? 'NH';
|
||||
const registrationNumber = org.registrationNumber ?? null;
|
||||
|
||||
const existing = await db
|
||||
if (registrationNumber != null) {
|
||||
const byRegNo = await db
|
||||
.select({ id: schema.orgs.id })
|
||||
.from(schema.orgs)
|
||||
.where(
|
||||
and(
|
||||
eq(schema.orgs.sourceRegistry, org.sourceRegistry),
|
||||
eq(schema.orgs.registrationNumber, registrationNumber),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
const found = byRegNo[0];
|
||||
if (found != null) {
|
||||
await db
|
||||
.update(schema.orgs)
|
||||
.set({
|
||||
name: org.name,
|
||||
city: org.city,
|
||||
state,
|
||||
registrationStatus: org.registrationStatus,
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where(eq(schema.orgs.id, found.id));
|
||||
return { id: found.id, inserted: false };
|
||||
}
|
||||
}
|
||||
|
||||
const byName = await db
|
||||
.select({ id: schema.orgs.id })
|
||||
.from(schema.orgs)
|
||||
.where(
|
||||
@@ -48,13 +84,15 @@ export async function serverUpsertOrgFromRegistry(
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
const found = existing[0];
|
||||
const found = byName[0];
|
||||
if (found != null) {
|
||||
await db
|
||||
.update(schema.orgs)
|
||||
.set({
|
||||
registrationStatus: org.registrationStatus,
|
||||
sourceRegistry: org.sourceRegistry,
|
||||
// Adopt the registry id so future re-scans match on the stable key.
|
||||
registrationNumber,
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where(eq(schema.orgs.id, found.id));
|
||||
@@ -67,6 +105,7 @@ export async function serverUpsertOrgFromRegistry(
|
||||
name: org.name,
|
||||
city: org.city,
|
||||
state,
|
||||
registrationNumber,
|
||||
registrationStatus: org.registrationStatus,
|
||||
sourceRegistry: org.sourceRegistry,
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, asc, isNull } from 'drizzle-orm';
|
||||
import { and, asc, isNull, sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
@@ -31,6 +31,12 @@ export async function serverListOrgsNeedingEnrichment(
|
||||
})
|
||||
.from(schema.orgs)
|
||||
.where(and(isNull(schema.orgs.ein), isNull(schema.orgs.totalRevenue)))
|
||||
.orderBy(asc(schema.orgs.updatedAt))
|
||||
// NH good-standing orgs are the ICP — enrich them before the long tail
|
||||
// of out-of-state registrants; oldest-touched first within each tier.
|
||||
.orderBy(
|
||||
sql`(${schema.orgs.state} = 'NH') DESC`,
|
||||
sql`(${schema.orgs.registrationStatus} = 'good_standing') DESC`,
|
||||
asc(schema.orgs.updatedAt),
|
||||
)
|
||||
.limit(limit);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user