diff --git a/apps/outreach-worker/.env.example b/apps/outreach-worker/.env.example index 8700a19..8542484 100644 --- a/apps/outreach-worker/.env.example +++ b/apps/outreach-worker/.env.example @@ -16,6 +16,9 @@ GCP_SERVICE_ACCOUNT_KEY_PATH=./secrets/gcp-service-account.json # NHDOJ Charitable Trusts registry PDF — the URL changes whenever NHDOJ # republishes (roughly monthly). When unset, the monthly ingestNhdojOrgs # workflow logs a warning and no-ops instead of failing. +# Local-file override for supervised runs / Akamai outages (takes +# precedence over the URL): +# NHDOJ_REGISTRY_PDF_PATH=./secrets/registered-charities.pdf # NHDOJ_REGISTRY_PDF_URL=https://www.doj.nh.gov/.../charitable-trusts-registry.pdf # Optional override for the Philanthropy News Digest RFP feed (defaults to diff --git a/apps/outreach-worker/src/run-once.ts b/apps/outreach-worker/src/run-once.ts new file mode 100644 index 0000000..918cf3b --- /dev/null +++ b/apps/outreach-worker/src/run-once.ts @@ -0,0 +1,120 @@ +/** + * Supervised one-off workflow runner. + * + * DATABASE_URL=... node --import tsx/esm src/run-once.ts [...] + * + * where is one or more of: ingestGrants, ingestPndRss, + * ingestNhdojOrgs, enrichOrgs, expireGrants — or `all` for the standard + * first-run order (grants → pnd → nhdoj → expire → enrich). + * + * Boots exactly like main.ts (same deps injection, same DBOS launch — see + * that file's boot-order comment), runs the requested workflows through + * their durable DBOS handles sequentially, then shuts down. Scheduled crons + * ARE active while this process lives; runs are short enough that this + * doesn't matter in practice. + */ +import { DBOS } from '@dbos-inc/dbos-sdk'; +import { DrizzleDataSource } from '@dbos-inc/drizzle-datasource'; +import { schema } from '@novelpad/outreach-core'; +import { drizzle } from 'drizzle-orm/node-postgres'; +import pg from 'pg'; + +import { runEnrichOrgsNow, setEnrichOrgsDeps } from './workflows/enrich-orgs.js'; +import { + runExpireGrantsNow, + setExpireGrantsDeps, +} from './workflows/expire-grants.js'; +import { + runIngestGrantsNow, + setIngestGrantsDeps, +} from './workflows/ingest-grants.js'; +import { + runIngestNhdojOrgsNow, + setIngestNhdojOrgsDeps, +} from './workflows/ingest-nhdoj-orgs.js'; +import { + runIngestPndRssNow, + setIngestPndRssDeps, +} from './workflows/ingest-pnd-rss.js'; + +const RUNNERS: Record Promise> = { + ingestGrants: runIngestGrantsNow, + ingestPndRss: runIngestPndRssNow, + ingestNhdojOrgs: runIngestNhdojOrgsNow, + enrichOrgs: runEnrichOrgsNow, + expireGrants: runExpireGrantsNow, +}; + +const FIRST_RUN_ORDER = [ + 'ingestGrants', + 'ingestPndRss', + 'ingestNhdojOrgs', + 'expireGrants', + 'enrichOrgs', +]; + +if (process.env.DATABASE_URL == null) { + throw new Error('run-once: DATABASE_URL is required'); +} + +const requested = process.argv.slice(2); +const names = requested.includes('all') ? FIRST_RUN_ORDER : requested; +if (names.length === 0 || names.some((n) => RUNNERS[n] == null)) { + console.error( + `Usage: run-once.ts \nKnown workflows: ${Object.keys(RUNNERS).join(', ')}, all`, + ); + process.exit(2); +} + +const { Pool } = pg; +const dbosClientConfig = { + connectionString: process.env.DATABASE_URL, + application_name: 'helmdocs-outreach-run-once', +} satisfies pg.ClientConfig; +const pool = new Pool({ + ...dbosClientConfig, + max: Number(process.env.PG_POOL_MAX ?? 20), +}); +const db = drizzle(pool, { schema }); + +async function main() { + await DrizzleDataSource.initializeDBOSSchema(dbosClientConfig); + + setIngestGrantsDeps({ db }); + setExpireGrantsDeps({ db }); + setIngestPndRssDeps({ db }); + setIngestNhdojOrgsDeps({ db }); + setEnrichOrgsDeps({ db }); + + DBOS.setConfig({ + name: 'helmdocs-outreach-worker', + systemDatabasePool: pool, + runAdminServer: false, + }); + await DBOS.launch(); + + let failed = false; + for (const name of names) { + console.log(`\n=== run-once: ${name} ===`); + const startedAt = Date.now(); + try { + await RUNNERS[name]!(); + console.log( + `=== run-once: ${name} OK in ${((Date.now() - startedAt) / 1000).toFixed(1)}s ===`, + ); + } catch (err) { + failed = true; + console.error(`=== run-once: ${name} FAILED ===`, err); + } + } + + // DBOS.shutdown() ends the pool we handed it via systemDatabasePool — + // a second pool.end() here throws "Called end on pool more than once". + await DBOS.shutdown(); + process.exit(failed ? 1 : 0); +} + +main().catch((err) => { + console.error('[run-once] fatal:', err); + process.exit(1); +}); diff --git a/apps/outreach-worker/src/sources/nhdoj/parse-registry.test.ts b/apps/outreach-worker/src/sources/nhdoj/parse-registry.test.ts index 144a13a..f91a3ac 100644 --- a/apps/outreach-worker/src/sources/nhdoj/parse-registry.test.ts +++ b/apps/outreach-worker/src/sources/nhdoj/parse-registry.test.ts @@ -5,236 +5,303 @@ import { normalizeRegistrationStatus, normalizeRegistryRows, reconstructRegistryRows, - type RegistryRow, } from './parse-registry.js'; -/** Column x-starts used across fixtures: name @50, city @300, status @450. */ -const NAME_X = 50; -const CITY_X = 300; -const STATUS_X = 450; +// x-anchors observed in the real PDF (2026-07-08 republish). +const X = { + regNo: 17, + name: 70, + address: 380, + city: 627, + state: 710, + zip: 748, + status: 820, + reportDue: 891, +} as const; function item(str: string, x: number, y: number): PositionedTextItem { return { str, x, y }; } -function headerRow(y: number): PositionedTextItem[] { +function headerLine(y: number): PositionedTextItem[] { return [ - item('Organization', NAME_X, y), - item('City', CITY_X, y), - item('Status', STATUS_X, y), + item('Reg. No.', X.regNo, y), + item('Charity Name', X.name, y), + item('Address', X.address, y), + item('City', X.city, y), + item('State', X.state, y), + item('Zip', X.zip, y), + item('Status', X.status, y), + item('Report Due', X.reportDue, y), + ]; +} + +function dataLine( + y: number, + cells: { + regNo?: string; + name?: string; + address?: string; + city?: string; + state?: string; + zip?: string; + status?: string; + reportDue?: string; + }, +): PositionedTextItem[] { + const out: PositionedTextItem[] = []; + if (cells.regNo) out.push(item(cells.regNo, X.regNo, y)); + if (cells.name) out.push(item(cells.name, X.name + 2, y)); + if (cells.address) out.push(item(cells.address, X.address, y)); + if (cells.city) out.push(item(cells.city, X.city, y)); + if (cells.state) out.push(item(cells.state, X.state, y)); + if (cells.zip) out.push(item(cells.zip, X.zip, y)); + if (cells.status) out.push(item(cells.status, X.status, y)); + if (cells.reportDue) out.push(item(cells.reportDue, X.reportDue, y)); + return out; +} + +function letterheadAndLegend(): PositionedTextItem[] { + return [ + item('New Hampshire Department of Justice', 15, 580), + item('Registered Charities List', 450, 581), + item('Charitable Trusts Unit', 896, 580), + item( + 'G = Good Standing; X = Not in Good Standing; S = Suspended', + 365, + 569, + ), ]; } describe('reconstructRegistryRows', () => { - it('returns no rows for an empty page', () => { - expect(reconstructRegistryRows([[]])).toEqual([]); - expect(reconstructRegistryRows([])).toEqual([]); - }); - - it('infers column boundaries from the header row and buckets data by x', () => { + it('parses a full page: letterhead, legend, header, data rows', () => { const page = [ - ...headerRow(900), - item('Acme Foundation', NAME_X, 880), - item('Concord', CITY_X, 880), - item('Good Standing', STATUS_X, 880), + ...letterheadAndLegend(), + ...headerLine(547), + ...dataLine(532, { + regNo: '35309', + name: '22ZERO Follow Me In', + address: 'PO Box 23', + city: 'Pulaski', + state: 'TN', + zip: '38478', + status: 'G', + reportDue: '5/15/2027', + }), + ...dataLine(518, { + regNo: '33454', + name: '#HappyPeriod', + address: '4911 Mountain View Drive', + city: 'Palmdale', + state: 'CA', + zip: '93552', + status: 'X', + reportDue: '9/14/2026', + }), + item('Updated: July 08, 2026', 15, 60), ]; const rows = reconstructRegistryRows([page]); - - expect(rows).toEqual([ - { name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' }, - ]); - }); - - it('infers boundaries correctly even when they differ from other fixtures', () => { - const page = [ - item('Organization', 40, 900), - item('City', 250, 900), - item('Status', 500, 900), - // Slightly right of each boundary — still buckets into the same column. - item('Bright Futures NH', 42, 870), - item('Nashua', 255, 870), - item('Active', 505, 870), - ]; - - const rows = reconstructRegistryRows([page]); - - expect(rows).toEqual([ - { name: 'Bright Futures NH', city: 'Nashua', status: 'Active' }, - ]); - }); - - it('folds a multi-line org name (continuation line has empty city/status) into the previous row', () => { - const page = [ - ...headerRow(900), - item('Very Long Nonprofit', NAME_X, 860), - item('Manchester', CITY_X, 860), - item('Lapsed', STATUS_X, 860), - // Wrapped second line of the same org name — no city/status text. - item('Name Incorporated', NAME_X, 840), - ]; - - const rows = reconstructRegistryRows([page]); - expect(rows).toEqual([ { - name: 'Very Long Nonprofit Name Incorporated', - city: 'Manchester', - status: 'Lapsed', + registrationNumber: '35309', + name: '22ZERO Follow Me In', + address: 'PO Box 23', + city: 'Pulaski', + state: 'TN', + zip: '38478', + status: 'G', + reportDue: '5/15/2027', + }, + { + registrationNumber: '33454', + name: '#HappyPeriod', + address: '4911 Mountain View Drive', + city: 'Palmdale', + state: 'CA', + zip: '93552', + status: 'X', + reportDue: '9/14/2026', }, ]); }); - it('supports multi-word continuation lines split across several text items', () => { + it('folds a wrapped name line into the previous row', () => { const page = [ - ...headerRow(900), - item('Friends of the', NAME_X, 860), - item('Merrimack', CITY_X, 860), - item('Current', STATUS_X, 860), - item('River', NAME_X, 840), - item('Watershed', NAME_X + 40, 840), + ...headerLine(547), + ...dataLine(532, { + regNo: '30456', + name: '1st New Hampshire Light Battery', + address: '11 Pinecrest Circle', + city: 'Bedford', + state: 'NH', + zip: '03110', + status: 'X', + }), + // Continuation: name column only — no reg no, no status. + ...dataLine(518, { name: 'Historical Association' }), ]; const rows = reconstructRegistryRows([page]); - - expect(rows).toEqual([ - { - name: 'Friends of the River Watershed', - city: 'Merrimack', - status: 'Current', - }, - ]); + expect(rows).toHaveLength(1); + expect(rows[0]!.name).toBe( + '1st New Hampshire Light Battery Historical Association', + ); }); - it('drops header, letterhead, and page-number footer lines', () => { + it('folds wrapped address text without disturbing the name', () => { const page = [ - item('State of New Hampshire Department of Justice', NAME_X, 950), - ...headerRow(900), - item('Acme Foundation', NAME_X, 880), - item('Concord', CITY_X, 880), - item('Good Standing', STATUS_X, 880), - item('3', NAME_X, 50), + ...headerLine(547), + ...dataLine(532, { + regNo: '32030', + name: '#WalkAway Foundation', + address: '10521 Judicial Drive, Suite 200-A', + city: 'Fairfax', + state: 'VA', + zip: '22030', + status: 'G', + }), + ...dataLine(518, { address: 'Fairfax, VA 22030' }), ]; const rows = reconstructRegistryRows([page]); - - expect(rows).toEqual([ - { name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' }, - ]); + expect(rows).toHaveLength(1); + expect(rows[0]!.name).toBe('#WalkAway Foundation'); + expect(rows[0]!.address).toBe( + '10521 Judicial Drive, Suite 200-A Fairfax, VA 22030', + ); }); - it('drops a "Page X of Y" footer line', () => { - const page = [ - ...headerRow(900), - item('Acme Foundation', NAME_X, 880), - item('Concord', CITY_X, 880), - item('Good Standing', STATUS_X, 880), - item('Page 1 of 12', NAME_X, 50), - ]; - - const rows = reconstructRegistryRows([page]); - - expect(rows).toEqual([ - { name: 'Acme Foundation', city: 'Concord', status: 'Good Standing' }, - ]); - }); - - it('accepts explicit columnBoundaries for a continuation page with no repeated header', () => { - const page = [ - item('Second Page Org', NAME_X, 900), - item('Keene', CITY_X, 900), - item('Unknown', STATUS_X, 900), - ]; - - const rows = reconstructRegistryRows([page], { - columnBoundaries: [NAME_X, CITY_X, STATUS_X], - }); - - expect(rows).toEqual([ - { name: 'Second Page Org', city: 'Keene', status: 'Unknown' }, - ]); - }); - - it('reuses the last inferred boundaries across pages whose header does not repeat', () => { + it('reuses the last-known anchors on header-less continuation pages', () => { const page1 = [ - ...headerRow(900), - item('First Page Org', NAME_X, 880), - item('Concord', CITY_X, 880), - item('Good Standing', STATUS_X, 880), + ...headerLine(547), + ...dataLine(532, { + regNo: '1', + name: 'Alpha', + city: 'Concord', + state: 'NH', + status: 'G', + }), ]; const page2 = [ - item('Second Page Org', NAME_X, 900), - item('Keene', CITY_X, 900), - item('Lapsed', STATUS_X, 900), + ...dataLine(532, { + regNo: '2', + name: 'Beta', + city: 'Nashua', + state: 'NH', + status: 'S', + }), ]; const rows = reconstructRegistryRows([page1, page2]); - - expect(rows).toEqual([ - { name: 'First Page Org', city: 'Concord', status: 'Good Standing' }, - { name: 'Second Page Org', city: 'Keene', status: 'Lapsed' }, - ]); + expect(rows.map((r) => r.name)).toEqual(['Alpha', 'Beta']); }); - it('throws when a page has data but no boundaries can be determined', () => { - const page = [item('Mystery Org', NAME_X, 900)]; + it('throws loudly when no anchors can be determined', () => { + const page = [ + ...dataLine(532, { regNo: '1', name: 'Alpha', status: 'G' }), + ]; + expect(() => reconstructRegistryRows([page])).toThrow(/column anchors/); + }); - expect(() => reconstructRegistryRows([page])).toThrow( - /could not determine column boundaries/, - ); + it('accepts explicit anchors and skips empty pages', () => { + const anchors = [ + X.regNo, + X.name, + X.address, + X.city, + X.state, + X.zip, + X.status, + X.reportDue, + ] as const; + const page = [ + ...dataLine(532, { regNo: '9', name: 'Gamma', status: 'G' }), + ]; + const rows = reconstructRegistryRows([[], page], { + columnAnchors: anchors, + }); + expect(rows).toEqual([ + { + registrationNumber: '9', + name: 'Gamma', + address: null, + city: null, + state: null, + zip: null, + status: 'G', + reportDue: null, + }, + ]); }); }); describe('normalizeRegistrationStatus', () => { - it.each([ - ['Good Standing', 'good_standing'], - ['GOOD STANDING', 'good_standing'], - ['Current', 'good_standing'], - ['Active', 'good_standing'], - ['Lapsed', 'lapsed'], - ['Delinquent', 'lapsed'], - ['Suspended', 'lapsed'], - ['Expired', 'lapsed'], - ['Revoked', 'lapsed'], - ['', 'unknown'], - ['Pending Review', 'unknown'], - ] as const)('maps %s -> %s', (input, expected) => { - expect(normalizeRegistrationStatus(input)).toBe(expected); + it('maps the registry letter codes', () => { + expect(normalizeRegistrationStatus('G')).toBe('good_standing'); + expect(normalizeRegistrationStatus('g')).toBe('good_standing'); + expect(normalizeRegistrationStatus('X')).toBe('lapsed'); + expect(normalizeRegistrationStatus('S')).toBe('suspended'); + }); + + it('maps spelled-out fallbacks, including the negated phrase', () => { + expect(normalizeRegistrationStatus('Good Standing')).toBe('good_standing'); + expect(normalizeRegistrationStatus('Not in Good Standing')).toBe('lapsed'); + expect(normalizeRegistrationStatus('Suspended')).toBe('suspended'); + expect(normalizeRegistrationStatus('Revoked')).toBe('lapsed'); + }); + + it('maps blanks and surprises to unknown', () => { + expect(normalizeRegistrationStatus('')).toBe('unknown'); + expect(normalizeRegistrationStatus(' ')).toBe('unknown'); + expect(normalizeRegistrationStatus('Q')).toBe('unknown'); }); }); describe('normalizeRegistryRows', () => { - it('maps status variants, trims/collapses whitespace, and preserves a null city', () => { - const rows: RegistryRow[] = [ - { name: ' Acme Foundation ', city: ' Concord ', status: 'Good Standing' }, - { name: 'Delinquent Org', city: null, status: 'Delinquent' }, - { name: 'Mystery Org', city: 'Keene', status: 'Something Else' }, - ]; - - expect(normalizeRegistryRows(rows)).toEqual([ - { name: 'Acme Foundation', city: 'Concord', status: 'good_standing' }, - { name: 'Delinquent Org', city: null, status: 'lapsed' }, - { name: 'Mystery Org', city: 'Keene', status: 'unknown' }, + it('trims to org fields, uppercases state, drops artifacts', () => { + const rows = normalizeRegistryRows([ + { + registrationNumber: '35309', + name: ' 22ZERO Follow Me In ', + address: 'PO Box 23', + city: ' Pulaski ', + state: 'tn', + zip: '38478', + status: 'G', + reportDue: '5/15/2027', + }, + { + registrationNumber: null, + name: 'Updated: July 08, 2026', + address: null, + city: null, + state: null, + zip: null, + status: '', + reportDue: null, + }, + { + registrationNumber: null, + name: '', + address: null, + city: null, + state: null, + zip: null, + status: 'G', + reportDue: null, + }, ]); - }); - it('drops rows with an empty name and header/footer artifact rows that slipped through', () => { - const rows: RegistryRow[] = [ - { name: '', city: 'Concord', status: 'Good Standing' }, - { name: 'Organization City Status', city: null, status: '' }, - { name: 'Real Org', city: 'Nashua', status: 'Active' }, - ]; - - expect(normalizeRegistryRows(rows)).toEqual([ - { name: 'Real Org', city: 'Nashua', status: 'good_standing' }, - ]); - }); - - it('collapses an empty-string city to null', () => { - const rows: RegistryRow[] = [{ name: 'Org', city: ' ', status: 'Active' }]; - - expect(normalizeRegistryRows(rows)).toEqual([ - { name: 'Org', city: null, status: 'good_standing' }, + expect(rows).toEqual([ + { + registrationNumber: '35309', + name: '22ZERO Follow Me In', + city: 'Pulaski', + state: 'TN', + status: 'good_standing', + }, ]); }); }); diff --git a/apps/outreach-worker/src/sources/nhdoj/parse-registry.ts b/apps/outreach-worker/src/sources/nhdoj/parse-registry.ts index 5b4a2a0..4916167 100644 --- a/apps/outreach-worker/src/sources/nhdoj/parse-registry.ts +++ b/apps/outreach-worker/src/sources/nhdoj/parse-registry.ts @@ -1,55 +1,92 @@ /** * PURE row reconstruction over positioned text extracted from the NHDOJ * Charitable Trusts registry PDF (see `extract-pdf-text.ts` for the impure - * layer that produces the input). No PDF or network dependency here — every - * function operates on plain `PositionedTextItem[][]` so tests can drive it - * with synthetic fixtures. + * layer). No PDF or network dependency here — every function operates on + * plain `PositionedTextItem[][]` so tests drive it with synthetic fixtures. * - * The registry renders as a 3-column table (organization name, city, - * registration status). pdf.js gives us a flat bag of positioned glyph runs - * per page with no row/column structure, so reconstruction happens in two - * passes: + * Layout (verified against the real PDF, "Registered Charities List", + * updated 2026-07-08, 427 pages): an 8-column table — * + * Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due + * + * with a legend line "G = Good Standing; X = Not in Good Standing; + * S = Suspended" under the letterhead, a repeated header row per page, and + * an "Updated: " footer. Status is a single letter (G/X/S). The list + * includes out-of-state charities registered to solicit in NH, so State is + * carried through rather than assumed 'NH'. + * + * Reconstruction: * 1. `reconstructRegistryRows` groups items into visual lines by y - * (tolerance ~2pt), buckets each line's items into columns by x - * (boundaries inferred from the header row's token x-positions, or - * supplied explicitly via `options.columnBoundaries` when a page's - * header doesn't repeat), and stitches multi-line org names back - * together — a continuation line has text only in the name column. - * 2. `normalizeRegistryRows` maps the free-text status column onto the - * tri-state the DB expects and drops anything that isn't really a data - * row (repeated header, page number, agency letterhead). + * (~2pt tolerance), buckets each line's items into the 8 columns by x + * (anchors inferred from the header row's token positions, or supplied + * via `options.columnAnchors`), and folds wrapped lines — a line with + * no Reg. No. and no Status continues the previous row's name/address. + * 2. `normalizeRegistryRows` maps G/X/S onto the status enum and drops + * artifact rows (legend, letterhead, page footers). */ import type { PositionedTextItem } from './extract-pdf-text.js'; export interface RegistryRow { + readonly registrationNumber: string | null; readonly name: string; + readonly address: string | null; readonly city: string | null; + readonly state: string | null; + readonly zip: string | null; readonly status: string; + readonly reportDue: string | null; } -export type NormalizedRegistrationStatus = 'good_standing' | 'lapsed' | 'unknown'; +export type NormalizedRegistrationStatus = + | 'good_standing' + | 'lapsed' + | 'suspended' + | 'unknown'; export interface NormalizedRegistryRow { + readonly registrationNumber: string | null; readonly name: string; readonly city: string | null; + readonly state: string | null; readonly status: NormalizedRegistrationStatus; } +/** Ascending x-anchors for the 8 columns, left edge of each. */ +export type ColumnAnchors = readonly [ + number, // Reg. No. + number, // Charity Name + number, // Address + number, // City + number, // State + number, // Zip + number, // Status + number, // Report Due +]; + export interface ParseRegistryOptions { - /** Max y-distance (pt) between items considered part of the same visual line. Default 2. */ + /** Max y-distance (pt) between items on the same visual line. Default 2. */ readonly yTolerance?: number; /** - * Explicit x-position column starts `[nameStart, cityStart, statusStart]`, - * ascending. When omitted, boundaries are inferred per page from that - * page's header row (falling back to the most recently inferred/ provided - * boundaries for pages whose header doesn't repeat, e.g. continuation - * pages). + * Explicit column anchors. When omitted, inferred per page from that + * page's header row; pages without a header reuse the last-known anchors. */ - readonly columnBoundaries?: readonly [number, number, number]; + readonly columnAnchors?: ColumnAnchors; } const DEFAULT_Y_TOLERANCE = 2; +/** Data cells sit within a few pt left of their header token's x. */ +const COLUMN_X_SLACK = 4; + +const HEADER_PATTERNS: readonly RegExp[] = [ + /^reg\.?\s*no\.?$/i, + /^charity\s+name$/i, + /^address$/i, + /^city$/i, + /^state$/i, + /^zip$/i, + /^status$/i, + /^report\s+due$/i, +]; interface Line { readonly y: number; @@ -60,10 +97,7 @@ function collapseWhitespace(text: string): string { return text.trim().replace(/\s+/g, ' '); } -/** - * Groups a page's positioned items into visual lines (top-to-bottom by y, - * items within a line ordered left-to-right by x). - */ +/** Groups items into visual lines: top-to-bottom, left-to-right. */ function groupLines( items: readonly PositionedTextItem[], yTolerance: number, @@ -88,33 +122,29 @@ function groupLines( return lines; } -/** Bucket an x-position into a column index given ascending column-start boundaries. */ -function columnIndexForX(x: number, boundaries: readonly number[]): number { - for (let i = boundaries.length - 1; i >= 0; i--) { - if (x >= boundaries[i]!) return i; +function columnIndexForX(x: number, anchors: ColumnAnchors): number { + for (let i = anchors.length - 1; i >= 0; i--) { + if (x >= anchors[i]! - COLUMN_X_SLACK) return i; } return 0; } /** - * Looks for a line whose items spell out the three column headers - * ("Organization"/"Name", "City", "Status") and returns their x-positions, - * ascending — which is also left-to-right table order (name, city, status). + * Finds the header row (all 8 column labels on one line) and returns each + * label's x-position as the column anchors. */ -function inferColumnBoundariesFromHeader( +function inferColumnAnchorsFromHeader( lines: readonly Line[], -): [number, number, number] | null { +): ColumnAnchors | null { for (const line of lines) { - const nameItem = line.items.find((i) => - /organi[sz]ation|^name$/i.test(i.str.trim()), - ); - const cityItem = line.items.find((i) => /^city$/i.test(i.str.trim())); - const statusItem = line.items.find((i) => /status/i.test(i.str.trim())); - - if (nameItem != null && cityItem != null && statusItem != null) { - return [nameItem.x, cityItem.x, statusItem.x].sort( - (a, b) => a - b, - ) as [number, number, number]; + const anchors: number[] = []; + for (const pattern of HEADER_PATTERNS) { + const hit = line.items.find((i) => pattern.test(i.str.trim())); + if (hit == null) break; + anchors.push(hit.x); + } + if (anchors.length === HEADER_PATTERNS.length) { + return [...anchors].sort((a, b) => a - b) as unknown as ColumnAnchors; } } return null; @@ -123,19 +153,23 @@ function inferColumnBoundariesFromHeader( function isColumnHeaderLine(lineText: string): boolean { const upper = lineText.toUpperCase(); return ( - (upper.includes('ORGANIZATION') || /\bNAME\b/.test(upper)) && + upper.includes('CHARITY NAME') && upper.includes('CITY') && upper.includes('STATUS') ); } -/** Page-number footers, repeated agency letterhead, and blank lines. */ +/** Legend, letterhead, page footers, and blank lines. */ function isFooterOrArtifactLine(lineText: string): boolean { const trimmed = lineText.trim(); if (trimmed === '') return true; if (/^\d+$/.test(trimmed)) return true; if (/^page\s+\d+(\s+of\s+\d+)?$/i.test(trimmed)) return true; if (/department of justice/i.test(trimmed)) return true; + if (/registered charities list/i.test(trimmed)) return true; + if (/charitable trusts unit/i.test(trimmed)) return true; + if (/=\s*good standing/i.test(trimmed)) return true; + if (/^updated:/i.test(trimmed)) return true; return false; } @@ -144,22 +178,19 @@ function isHeaderOrFooterLine(lineText: string): boolean { } /** - * Reconstructs `{ name, city, status }` rows from positioned text, one page - * array at a time. Multi-line org names (a continuation line with nothing - * in the city/status columns) are folded back into the previous row. + * Reconstructs registry rows from positioned text, one page at a time. + * Wrapped rows (no Reg. No., no Status) fold their name/address text back + * into the previous row. * - * Throws if a page has data but no column boundaries can be determined - * (no header row found on any page so far, and none supplied) — that means - * the registry's layout changed and silent misparsing is worse than a loud - * failure here. + * Throws if a page has data but no column anchors can be determined — + * a layout change should fail loudly, not misparse silently. */ export function reconstructRegistryRows( pages: readonly (readonly PositionedTextItem[])[], options: ParseRegistryOptions = {}, ): RegistryRow[] { const yTolerance = options.yTolerance ?? DEFAULT_Y_TOLERANCE; - let boundaries: readonly [number, number, number] | null = - options.columnBoundaries ?? null; + let anchors: ColumnAnchors | null = options.columnAnchors ?? null; const rows: RegistryRow[] = []; @@ -168,15 +199,15 @@ export function reconstructRegistryRows( const lines = groupLines(pageItems, yTolerance); - if (options.columnBoundaries == null) { - const inferred = inferColumnBoundariesFromHeader(lines); - if (inferred != null) boundaries = inferred; + if (options.columnAnchors == null) { + const inferred = inferColumnAnchorsFromHeader(lines); + if (inferred != null) anchors = inferred; } - if (boundaries == null) { + if (anchors == null) { throw new Error( - 'reconstructRegistryRows: could not determine column boundaries ' + - '(no header row found and none provided via options.columnBoundaries)', + 'reconstructRegistryRows: could not determine column anchors ' + + '(no header row found and none provided via options.columnAnchors)', ); } @@ -184,60 +215,97 @@ export function reconstructRegistryRows( const lineText = line.items.map((i) => i.str).join(' '); if (isHeaderOrFooterLine(lineText)) continue; - const columns: [string[], string[], string[]] = [[], [], []]; + const columns: string[][] = Array.from( + { length: HEADER_PATTERNS.length }, + () => [], + ); for (const item of line.items) { - const idx = columnIndexForX(item.x, boundaries); - columns[idx as 0 | 1 | 2].push(item.str); + if (item.str.trim() === '') continue; + columns[columnIndexForX(item.x, anchors)]!.push(item.str); } - const name = collapseWhitespace(columns[0].join(' ')); - const city = collapseWhitespace(columns[1].join(' ')); - const status = collapseWhitespace(columns[2].join(' ')); + const cell = (i: number): string => + collapseWhitespace(columns[i]!.join(' ')); - if (name === '' && city === '' && status === '') continue; + const registrationNumber = cell(0); + const name = cell(1); + const address = cell(2); + const city = cell(3); + const state = cell(4); + const zip = cell(5); + const status = cell(6); + const reportDue = cell(7); - // A line with text only in the name column continues the previous - // row's (wrapped) org name rather than starting a new row. - if (city === '' && status === '' && rows.length > 0) { + if ( + [registrationNumber, name, address, city, state, zip, status, reportDue] + .every((c) => c === '') + ) { + continue; + } + + // A wrapped row: no registry number and no status — its name/address + // text continues the previous row rather than starting a new one. + if (registrationNumber === '' && status === '' && rows.length > 0) { const prev = rows[rows.length - 1]!; rows[rows.length - 1] = { ...prev, name: collapseWhitespace(`${prev.name} ${name}`), + address: + address === '' + ? prev.address + : collapseWhitespace(`${prev.address ?? ''} ${address}`), }; continue; } - rows.push({ name, city: city === '' ? null : city, status }); + rows.push({ + registrationNumber: registrationNumber === '' ? null : registrationNumber, + name, + address: address === '' ? null : address, + city: city === '' ? null : city, + state: state === '' ? null : state, + zip: zip === '' ? null : zip, + status, + reportDue: reportDue === '' ? null : reportDue, + }); } } return rows; } -const GOOD_STANDING_PATTERNS = [/good\s*standing/i, /\bcurrent\b/i, /\bactive\b/i]; -const LAPSED_PATTERNS = [ - /\blapsed\b/i, - /\bdelinquent\b/i, - /\bsuspended\b/i, - /\bexpired\b/i, - /\brevoked\b/i, -]; - +/** + * Registry legend: G = Good Standing; X = Not in Good Standing; + * S = Suspended. Phrase forms accepted as a fallback for older + * republishes that spelled statuses out. + */ export function normalizeRegistrationStatus( status: string, ): NormalizedRegistrationStatus { const trimmed = status.trim(); if (trimmed === '') return 'unknown'; - if (GOOD_STANDING_PATTERNS.some((p) => p.test(trimmed))) return 'good_standing'; - if (LAPSED_PATTERNS.some((p) => p.test(trimmed))) return 'lapsed'; + if (/^g$/i.test(trimmed) || /good\s*standing/i.test(trimmed)) { + // "Not in Good Standing" also contains the phrase — X handles the real + // file; this phrase fallback must not swallow the negated form. + if (/not\s+in\s+good/i.test(trimmed)) return 'lapsed'; + return 'good_standing'; + } + if (/^s$/i.test(trimmed) || /\bsuspended\b/i.test(trimmed)) return 'suspended'; + if ( + /^x$/i.test(trimmed) || + /\blapsed\b/i.test(trimmed) || + /\bdelinquent\b/i.test(trimmed) || + /\bexpired\b/i.test(trimmed) || + /\brevoked\b/i.test(trimmed) + ) { + return 'lapsed'; + } return 'unknown'; } /** - * Maps raw rows onto the tri-state status the DB expects, collapses - * whitespace, and drops anything that isn't really a registrant (a - * repeated header row or letterhead line that slipped through row - * reconstruction as a degenerate one-column row). + * Maps raw rows onto the DB's status enum, trims to the fields the org + * table stores, and drops anything that isn't really a registrant. */ export function normalizeRegistryRows( rows: readonly RegistryRow[], @@ -249,9 +317,14 @@ export function normalizeRegistryRows( if (name === '' || isHeaderOrFooterLine(name)) continue; const city = row.city == null ? null : collapseWhitespace(row.city); + const state = + row.state == null ? null : collapseWhitespace(row.state).toUpperCase(); + normalized.push({ + registrationNumber: row.registrationNumber, name, city: city === '' ? null : city, + state: state === '' ? null : state, status: normalizeRegistrationStatus(row.status), }); } diff --git a/apps/outreach-worker/src/sources/propublica/client.ts b/apps/outreach-worker/src/sources/propublica/client.ts index 301d6ff..8564c7b 100644 --- a/apps/outreach-worker/src/sources/propublica/client.ts +++ b/apps/outreach-worker/src/sources/propublica/client.ts @@ -62,6 +62,13 @@ export async function searchOrganizations( url.searchParams.set('state[id]', state); const res = await fetch(url); + if (res.status === 404) { + // Nonprofit Explorer quirk (verified 2026-07-16): a filtered search + // with zero hits responds 404, not 200-with-empty-list. Zero hits is a + // legitimate "no match" — the org resolves as unresolved, not failed. + await wait(options.politenessDelayMs ?? DEFAULT_POLITENESS_DELAY_MS); + return { organizations: [] }; + } if (!res.ok) { throw new Error( `ProPublica search failed for "${name}" (${state}): ${res.status} ${res.statusText}`, diff --git a/apps/outreach-worker/src/workflows/enrich-orgs.ts b/apps/outreach-worker/src/workflows/enrich-orgs.ts index 4bd80ce..8087d88 100644 --- a/apps/outreach-worker/src/workflows/enrich-orgs.ts +++ b/apps/outreach-worker/src/workflows/enrich-orgs.ts @@ -181,6 +181,7 @@ async function runEnrichOrgs(): Promise { const g = globalThis as unknown as { __outreachEnrichOrgsRegistered?: boolean; + __outreachEnrichOrgsHandle?: (scheduledTime: Date, startedAt: Date) => Promise; }; if (!g.__outreachEnrichOrgsRegistered) { @@ -197,10 +198,26 @@ if (!g.__outreachEnrichOrgsRegistered) { // Must be registered as BOTH a workflow and a scheduled function, // referencing the same function object — see module doc comment. - DBOS.registerWorkflow(enrichOrgs, { name: 'enrichOrgs' }); + g.__outreachEnrichOrgsHandle = DBOS.registerWorkflow(enrichOrgs, { + name: 'enrichOrgs', + }); DBOS.registerScheduled(enrichOrgs, { crontab: '0 5 * * *', name: 'enrichOrgs', mode: SchedulerMode.ExactlyOncePerInterval, }); } + +/** + * Starts one durable run of this workflow immediately through DBOS — + * the exact production path (workflow + checkpointed steps), used by + * `run-once.ts` for supervised/manual passes. Requires deps injected and + * `DBOS.launch()` completed. + */ +export function runEnrichOrgsNow(): Promise { + const handle = g.__outreachEnrichOrgsHandle; + if (handle == null) { + throw new Error('enrichOrgs is not registered; was this module imported before DBOS.launch()?'); + } + return handle(new Date(), new Date()); +} diff --git a/apps/outreach-worker/src/workflows/expire-grants.ts b/apps/outreach-worker/src/workflows/expire-grants.ts index 3515447..834e9d7 100644 --- a/apps/outreach-worker/src/workflows/expire-grants.ts +++ b/apps/outreach-worker/src/workflows/expire-grants.ts @@ -47,6 +47,7 @@ async function runExpireGrants(): Promise { const g = globalThis as unknown as { __outreachExpireGrantsRegistered?: boolean; + __outreachExpireGrantsHandle?: (scheduledTime: Date, startedAt: Date) => Promise; }; if (!g.__outreachExpireGrantsRegistered) { @@ -61,10 +62,26 @@ if (!g.__outreachExpireGrantsRegistered) { } }; - DBOS.registerWorkflow(expireGrants, { name: 'expireGrants' }); + g.__outreachExpireGrantsHandle = DBOS.registerWorkflow(expireGrants, { + name: 'expireGrants', + }); DBOS.registerScheduled(expireGrants, { crontab: '0 * * * *', name: 'expireGrants', mode: SchedulerMode.ExactlyOncePerInterval, }); } + +/** + * Starts one durable run of this workflow immediately through DBOS — + * the exact production path (workflow + checkpointed steps), used by + * `run-once.ts` for supervised/manual passes. Requires deps injected and + * `DBOS.launch()` completed. + */ +export function runExpireGrantsNow(): Promise { + const handle = g.__outreachExpireGrantsHandle; + if (handle == null) { + throw new Error('expireGrants is not registered; was this module imported before DBOS.launch()?'); + } + return handle(new Date(), new Date()); +} diff --git a/apps/outreach-worker/src/workflows/ingest-grants.ts b/apps/outreach-worker/src/workflows/ingest-grants.ts index f613576..009f8d8 100644 --- a/apps/outreach-worker/src/workflows/ingest-grants.ts +++ b/apps/outreach-worker/src/workflows/ingest-grants.ts @@ -154,6 +154,7 @@ async function runIngestGrants(): Promise { const g = globalThis as unknown as { __outreachIngestGrantsRegistered?: boolean; + __outreachIngestGrantsHandle?: (scheduledTime: Date, startedAt: Date) => Promise; }; if (!g.__outreachIngestGrantsRegistered) { @@ -170,10 +171,26 @@ if (!g.__outreachIngestGrantsRegistered) { // Must be registered as BOTH a workflow and a scheduled function, // referencing the same function object — see module doc comment. - DBOS.registerWorkflow(ingestGrants, { name: 'ingestGrants' }); + g.__outreachIngestGrantsHandle = DBOS.registerWorkflow(ingestGrants, { + name: 'ingestGrants', + }); DBOS.registerScheduled(ingestGrants, { crontab: '0 3 * * *', name: 'ingestGrants', mode: SchedulerMode.ExactlyOncePerInterval, }); } + +/** + * Starts one durable run of this workflow immediately through DBOS — + * the exact production path (workflow + checkpointed steps), used by + * `run-once.ts` for supervised/manual passes. Requires deps injected and + * `DBOS.launch()` completed. + */ +export function runIngestGrantsNow(): Promise { + const handle = g.__outreachIngestGrantsHandle; + if (handle == null) { + throw new Error('ingestGrants is not registered; was this module imported before DBOS.launch()?'); + } + return handle(new Date(), new Date()); +} diff --git a/apps/outreach-worker/src/workflows/ingest-nhdoj-orgs.ts b/apps/outreach-worker/src/workflows/ingest-nhdoj-orgs.ts index bef0224..40381d2 100644 --- a/apps/outreach-worker/src/workflows/ingest-nhdoj-orgs.ts +++ b/apps/outreach-worker/src/workflows/ingest-nhdoj-orgs.ts @@ -50,12 +50,29 @@ function getIngestNhdojOrgsDeps(): IngestNhdojOrgsDeps { } async function fetchNhdojRegistryPdf(url: string): Promise { - const res = await fetch(url); + // mm.nh.gov sits behind Akamai bot detection: bare curl/default clients + // get a 403 "Access Denied" HTML page. Node's fetch passes with + // browser-like headers (verified 2026-07-16). + const res = await fetch(url, { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', + Accept: 'application/pdf,*/*', + }, + }); if (!res.ok) { throw new Error( `fetchNhdojRegistryPdf: ${url} responded ${res.status} ${res.statusText}`, ); } + const contentType = res.headers.get('content-type') ?? ''; + if (!contentType.includes('pdf')) { + // Akamai serves its denial as 200 text/html through some paths — treat + // a non-PDF body as a failure rather than feeding HTML to pdfjs. + throw new Error( + `fetchNhdojRegistryPdf: expected a PDF but got content-type "${contentType}" from ${url}`, + ); + } const buf = await res.arrayBuffer(); return new Uint8Array(buf); } @@ -72,6 +89,10 @@ async function upsertRegistryOrg( return serverUpsertOrgFromRegistry(db, { name: row.name, city: row.city, + // The registry includes out-of-state charities registered to solicit + // in NH — carry their real state so the geography gate stays honest. + state: row.state, + registrationNumber: row.registrationNumber, registrationStatus: row.status, sourceRegistry: SOURCE_REGISTRY, }); @@ -85,18 +106,25 @@ const upsertRegistryOrgStep = DBOS.registerStep(upsertRegistryOrg, { async function runIngestNhdojOrgs(): Promise { const { db } = getIngestNhdojOrgsDeps(); + const pdfPath = process.env.NHDOJ_REGISTRY_PDF_PATH; const pdfUrl = process.env.NHDOJ_REGISTRY_PDF_URL; - if (pdfUrl == null || pdfUrl.trim() === '') { + + let pdfBytes: Uint8Array; + if (pdfPath != null && pdfPath.trim() !== '') { + // Local-file escape hatch for supervised runs and Akamai outages. + const { readFile } = await import('node:fs/promises'); + pdfBytes = new Uint8Array(await readFile(pdfPath)); + } else if (pdfUrl != null && pdfUrl.trim() !== '') { + pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl); + } else { // The registry PDF's URL changes whenever NHDOJ republishes it (no // stable endpoint) — a hard failure here would page on-call for a // config gap rather than a real outage, so skip quietly instead. console.warn( - '[ingest-nhdoj-orgs] NHDOJ_REGISTRY_PDF_URL is not set; skipping this pass.', + '[ingest-nhdoj-orgs] neither NHDOJ_REGISTRY_PDF_PATH nor NHDOJ_REGISTRY_PDF_URL is set; skipping this pass.', ); return; } - - const pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl); const pages = await extractPositionedText(pdfBytes); const rawRows = reconstructRegistryRows(pages); const normalizedRows = normalizeRegistryRows(rawRows); @@ -116,6 +144,7 @@ async function runIngestNhdojOrgs(): Promise { const g = globalThis as unknown as { __outreachIngestNhdojOrgsRegistered?: boolean; + __outreachIngestNhdojOrgsHandle?: (scheduledTime: Date, startedAt: Date) => Promise; }; if (!g.__outreachIngestNhdojOrgsRegistered) { @@ -132,10 +161,26 @@ if (!g.__outreachIngestNhdojOrgsRegistered) { // Must be registered as BOTH a workflow and a scheduled function, // referencing the same function object — see module doc comment. - DBOS.registerWorkflow(ingestNhdojOrgs, { name: 'ingestNhdojOrgs' }); + g.__outreachIngestNhdojOrgsHandle = DBOS.registerWorkflow(ingestNhdojOrgs, { + name: 'ingestNhdojOrgs', + }); DBOS.registerScheduled(ingestNhdojOrgs, { crontab: '0 4 1 * *', name: 'ingestNhdojOrgs', mode: SchedulerMode.ExactlyOncePerInterval, }); } + +/** + * Starts one durable run of this workflow immediately through DBOS — + * the exact production path (workflow + checkpointed steps), used by + * `run-once.ts` for supervised/manual passes. Requires deps injected and + * `DBOS.launch()` completed. + */ +export function runIngestNhdojOrgsNow(): Promise { + const handle = g.__outreachIngestNhdojOrgsHandle; + if (handle == null) { + throw new Error('ingestNhdojOrgs is not registered; was this module imported before DBOS.launch()?'); + } + return handle(new Date(), new Date()); +} diff --git a/apps/outreach-worker/src/workflows/ingest-pnd-rss.ts b/apps/outreach-worker/src/workflows/ingest-pnd-rss.ts index 56bd3c0..8dee0ec 100644 --- a/apps/outreach-worker/src/workflows/ingest-pnd-rss.ts +++ b/apps/outreach-worker/src/workflows/ingest-pnd-rss.ts @@ -88,6 +88,7 @@ async function runIngestPndRss(): Promise { const g = globalThis as unknown as { __outreachIngestPndRssRegistered?: boolean; + __outreachIngestPndRssHandle?: (scheduledTime: Date, startedAt: Date) => Promise; }; if (!g.__outreachIngestPndRssRegistered) { @@ -104,10 +105,26 @@ if (!g.__outreachIngestPndRssRegistered) { // Must be registered as BOTH a workflow and a scheduled function, // referencing the same function object — see module doc comment. - DBOS.registerWorkflow(ingestPndRss, { name: 'ingestPndRss' }); + g.__outreachIngestPndRssHandle = DBOS.registerWorkflow(ingestPndRss, { + name: 'ingestPndRss', + }); DBOS.registerScheduled(ingestPndRss, { crontab: '30 3 * * *', name: 'ingestPndRss', mode: SchedulerMode.ExactlyOncePerInterval, }); } + +/** + * Starts one durable run of this workflow immediately through DBOS — + * the exact production path (workflow + checkpointed steps), used by + * `run-once.ts` for supervised/manual passes. Requires deps injected and + * `DBOS.launch()` completed. + */ +export function runIngestPndRssNow(): Promise { + const handle = g.__outreachIngestPndRssHandle; + if (handle == null) { + throw new Error('ingestPndRss is not registered; was this module imported before DBOS.launch()?'); + } + return handle(new Date(), new Date()); +} diff --git a/docs/features/ingestion.md b/docs/features/ingestion.md index 3e38039..834b5fa 100644 --- a/docs/features/ingestion.md +++ b/docs/features/ingestion.md @@ -47,6 +47,13 @@ Grant upserts key on `grants.source_url`; org registry upserts key on case-insen ### Philanthropy News Digest RSS +> **Status (2026-07-16): feed retired upstream.** Every historical feed path +> (`/feeds/rfps.rss` and variants) now returns the site's HTML shell — PND is +> a Next.js app with client-side data and no `` feeds. +> The workflow runs and upserts zero rows. Rework candidate: drive their +> internal JSON API / `__NEXT_DATA__` instead. Low priority — PND was "cheap +> incremental coverage"; Grants.gov + NH state sources carry the corpus. + Nightly ingestion of the Philanthropy News Digest "RFPs" RSS feed (`https://philanthropynewsdigest.org/feeds/rfps.rss`, overridable via `PND_RFP_FEED_URL`). - **Client** — `src/sources/pnd-rss/client.ts`: `fetchPndRfpFeed(feedUrl)` does a plain `fetch`, throwing on any non-2xx response. @@ -94,3 +101,10 @@ Monthly re-scan of the NH Department of Justice Charitable Trusts Unit's registr // ... setIngestNhdojOrgsDeps({ db }); ``` + +## First-run field findings (2026-07-16) + +- **NHDOJ registry**: 8-column layout (`Reg. No. | Charity Name | Address | City | State | Zip | Status | Report Due`), single-letter statuses (G/X/S legend), includes out-of-state charities registered to solicit in NH. Parsed 13,709 registrants → 13,632 orgs (6,266 NH). `Reg. No.` is the stable upsert key (`orgs.registration_number`). mm.nh.gov sits behind Akamai: bare curl gets 403; Node fetch with browser-like headers passes (client sends them). `NHDOJ_REGISTRY_PDF_PATH` overrides the URL for supervised runs. +- **ProPublica**: a state-scoped search with zero hits returns **404**, not an empty list — the client maps 404 → no candidates (org resolves `unresolved`). Before the fix this tripped the 20% systemic-failure breaker. +- **Grants.gov**: first pass ingested 200 opportunities (search cap 1000 / detail cap 200); the hourly expiry sweep correctly expired a same-day deadline immediately. +- **Enrichment queue** prioritizes NH + good-standing orgs before the out-of-state tail; ~60% of NH orgs resolve an EIN per batch, the rest record an all-null attempt. diff --git a/packages/outreach-core/drizzle/server/1784231856_nhdoj-registry-fields.sql b/packages/outreach-core/drizzle/server/1784231856_nhdoj-registry-fields.sql new file mode 100644 index 0000000..0e52916 --- /dev/null +++ b/packages/outreach-core/drizzle/server/1784231856_nhdoj-registry-fields.sql @@ -0,0 +1,3 @@ +ALTER TYPE "public"."registration_status" ADD VALUE IF NOT EXISTS 'suspended' BEFORE 'unknown';--> statement-breakpoint +ALTER TABLE "orgs" ADD COLUMN "registration_number" text;--> statement-breakpoint +CREATE UNIQUE INDEX "idx_orgs_registry_reg_no" ON "orgs" USING btree ("source_registry","registration_number") WHERE "orgs"."source_registry" IS NOT NULL AND "orgs"."registration_number" IS NOT NULL; \ No newline at end of file diff --git a/packages/outreach-core/drizzle/server/meta/1784231856_snapshot.json b/packages/outreach-core/drizzle/server/meta/1784231856_snapshot.json new file mode 100644 index 0000000..f201172 --- /dev/null +++ b/packages/outreach-core/drizzle/server/meta/1784231856_snapshot.json @@ -0,0 +1,1149 @@ +{ + "id": "97041ebb-09e1-40dd-9042-c10af09a02c6", + "prevId": "ea850bc6-43a1-4ce1-88d4-dbac85993138", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.contacts": { + "name": "contacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_status": { + "name": "email_status", + "type": "email_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unverified'" + }, + "source_provider": { + "name": "source_provider", + "type": "contact_source_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "contact_priority", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'generic'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_contacts_org": { + "name": "idx_contacts_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_contacts_email": { + "name": "idx_contacts_email", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "contacts_org_id_orgs_id_fk": { + "name": "contacts_org_id_orgs_id_fk", + "tableFrom": "contacts", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grants": { + "name": "grants", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "funder": { + "name": "funder", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "synopsis": { + "name": "synopsis", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "eligibility_entity_types": { + "name": "eligibility_entity_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "geographic_scope": { + "name": "geographic_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "program_areas": { + "name": "program_areas", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "award_floor": { + "name": "award_floor", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "award_ceiling": { + "name": "award_ceiling", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "expected_awards_count": { + "name": "expected_awards_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "open_date": { + "name": "open_date", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_date": { + "name": "close_date", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "match_requirement": { + "name": "match_requirement", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "application_effort_estimate": { + "name": "application_effort_estimate", + "type": "application_effort_estimate", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "application_form_supported": { + "name": "application_form_supported", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "grant_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "grant_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "synopsis_embedding": { + "name": "synopsis_embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "last_verified_at": { + "name": "last_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_grants_source_url": { + "name": "idx_grants_source_url", + "columns": [ + { + "expression": "source_url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_grants_status": { + "name": "idx_grants_status", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_grants_close_date": { + "name": "idx_grants_close_date", + "columns": [ + { + "expression": "close_date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_grants_source": { + "name": "idx_grants_source", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "grants_synopsis_embedding_idx": { + "name": "grants_synopsis_embedding_idx", + "columns": [ + { + "expression": "synopsis_embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.matches": { + "name": "matches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "total_score": { + "name": "total_score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "subscores": { + "name": "subscores", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hard_gates_passed": { + "name": "hard_gates_passed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "easy_win": { + "name": "easy_win", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "rationale": { + "name": "rationale", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "review_status": { + "name": "review_status", + "type": "match_review_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "reject_reason": { + "name": "reject_reason", + "type": "match_reject_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "is_hero": { + "name": "is_hero", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_matches_org": { + "name": "idx_matches_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_matches_grant": { + "name": "idx_matches_grant", + "columns": [ + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_matches_review_status": { + "name": "idx_matches_review_status", + "columns": [ + { + "expression": "review_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "matches_org_id_orgs_id_fk": { + "name": "matches_org_id_orgs_id_fk", + "tableFrom": "matches", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "matches_grant_id_grants_id_fk": { + "name": "matches_grant_id_grants_id_fk", + "tableFrom": "matches", + "tableTo": "grants", + "columnsFrom": [ + "grant_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_profiles": { + "name": "org_profiles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "mission_statement": { + "name": "mission_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "programs": { + "name": "programs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "service_geography": { + "name": "service_geography", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recent_news": { + "name": "recent_news", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "known_funders": { + "name": "known_funders", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "staff": { + "name": "staff", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "budget_band": { + "name": "budget_band", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sources": { + "name": "sources", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "profile_embedding": { + "name": "profile_embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_org_profiles_org": { + "name": "idx_org_profiles_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_profiles_embedding_idx": { + "name": "org_profiles_embedding_idx", + "columns": [ + { + "expression": "profile_embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": { + "org_profiles_org_id_orgs_id_fk": { + "name": "org_profiles_org_id_orgs_id_fk", + "tableFrom": "org_profiles", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.orgs": { + "name": "orgs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "city": { + "name": "city", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'NH'" + }, + "ein": { + "name": "ein", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ntee_code": { + "name": "ntee_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "total_revenue": { + "name": "total_revenue", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "fiscal_year_end": { + "name": "fiscal_year_end", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_number": { + "name": "registration_number", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_status": { + "name": "registration_status", + "type": "registration_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "icp_band": { + "name": "icp_band", + "type": "icp_band", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "source_registry": { + "name": "source_registry", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_orgs_ein": { + "name": "idx_orgs_ein", + "columns": [ + { + "expression": "ein", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"orgs\".\"ein\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_orgs_registry_reg_no": { + "name": "idx_orgs_registry_reg_no", + "columns": [ + { + "expression": "source_registry", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"orgs\".\"source_registry\" IS NOT NULL AND \"orgs\".\"registration_number\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_orgs_icp_band": { + "name": "idx_orgs_icp_band", + "columns": [ + { + "expression": "icp_band", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_orgs_state": { + "name": "idx_orgs_state", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pipeline_events": { + "name": "pipeline_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "contact_id": { + "name": "contact_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "match_id": { + "name": "match_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "pipeline_event_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_pipeline_events_org": { + "name": "idx_pipeline_events_org", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_pipeline_events_contact": { + "name": "idx_pipeline_events_contact", + "columns": [ + { + "expression": "contact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_pipeline_events_match": { + "name": "idx_pipeline_events_match", + "columns": [ + { + "expression": "match_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_pipeline_events_type_occurred": { + "name": "idx_pipeline_events_type_occurred", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pipeline_events_org_id_orgs_id_fk": { + "name": "pipeline_events_org_id_orgs_id_fk", + "tableFrom": "pipeline_events", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "pipeline_events_contact_id_contacts_id_fk": { + "name": "pipeline_events_contact_id_contacts_id_fk", + "tableFrom": "pipeline_events", + "tableTo": "contacts", + "columnsFrom": [ + "contact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "pipeline_events_match_id_matches_id_fk": { + "name": "pipeline_events_match_id_matches_id_fk", + "tableFrom": "pipeline_events", + "tableTo": "matches", + "columnsFrom": [ + "match_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.application_effort_estimate": { + "name": "application_effort_estimate", + "schema": "public", + "values": [ + "loi_only", + "short_form", + "full_federal", + "unknown" + ] + }, + "public.contact_priority": { + "name": "contact_priority", + "schema": "public", + "values": [ + "named", + "generic" + ] + }, + "public.contact_source_provider": { + "name": "contact_source_provider", + "schema": "public", + "values": [ + "apollo", + "irs_990", + "website", + "manual" + ] + }, + "public.email_status": { + "name": "email_status", + "schema": "public", + "values": [ + "unverified", + "valid", + "risky", + "invalid" + ] + }, + "public.grant_source": { + "name": "grant_source", + "schema": "public", + "values": [ + "grants_gov", + "nh_state", + "irs_990pf", + "pnd_rss", + "candid", + "manual" + ] + }, + "public.grant_status": { + "name": "grant_status", + "schema": "public", + "values": [ + "open", + "expired", + "closed" + ] + }, + "public.icp_band": { + "name": "icp_band", + "schema": "public", + "values": [ + "below", + "primary", + "above", + "unknown" + ] + }, + "public.match_reject_reason": { + "name": "match_reject_reason", + "schema": "public", + "values": [ + "wrong_eligibility", + "wrong_geography", + "bad_capacity_fit", + "weak_mission_fit", + "stale_deadline", + "bad_contact", + "other" + ] + }, + "public.match_review_status": { + "name": "match_review_status", + "schema": "public", + "values": [ + "pending", + "approved", + "rejected", + "edited" + ] + }, + "public.pipeline_event_type": { + "name": "pipeline_event_type", + "schema": "public", + "values": [ + "enrolled", + "sent", + "opened", + "replied", + "bounced", + "unsubscribed", + "brief_requested", + "brief_sent", + "demo_booked", + "demo_held", + "pilot_started", + "converted" + ] + }, + "public.registration_status": { + "name": "registration_status", + "schema": "public", + "values": [ + "good_standing", + "lapsed", + "suspended", + "unknown" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/outreach-core/drizzle/server/meta/_journal.json b/packages/outreach-core/drizzle/server/meta/_journal.json index 667293d..11dff56 100644 --- a/packages/outreach-core/drizzle/server/meta/_journal.json +++ b/packages/outreach-core/drizzle/server/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1784214387212, "tag": "1784214387_initial-schema", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1784231856865, + "tag": "1784231856_nhdoj-registry-fields", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/outreach-core/src/db/schema.ts b/packages/outreach-core/src/db/schema.ts index 05c8180..d30486a 100644 --- a/packages/outreach-core/src/db/schema.ts +++ b/packages/outreach-core/src/db/schema.ts @@ -43,6 +43,7 @@ export const grantStatusEnum = pgEnum('grant_status', [ export const registrationStatusEnum = pgEnum('registration_status', [ 'good_standing', 'lapsed', + 'suspended', 'unknown', ]); @@ -174,6 +175,9 @@ export const orgs = pgTable( nteeCode: text('ntee_code'), totalRevenue: integer('total_revenue'), fiscalYearEnd: text('fiscal_year_end'), + // Registry-assigned identifier (e.g. NHDOJ "Reg. No.") — stable across + // republishes, unlike org names; the preferred upsert key when present. + registrationNumber: text('registration_number'), registrationStatus: registrationStatusEnum('registration_status') .notNull() .default('unknown'), @@ -186,6 +190,11 @@ export const orgs = pgTable( // Partial unique index: EIN is unique when present, but many small // orgs in early ingestion won't have one resolved yet. uniqueIndex('idx_orgs_ein').on(t.ein).where(sql`${t.ein} IS NOT NULL`), + uniqueIndex('idx_orgs_registry_reg_no') + .on(t.sourceRegistry, t.registrationNumber) + .where( + sql`${t.sourceRegistry} IS NOT NULL AND ${t.registrationNumber} IS NOT NULL`, + ), index('idx_orgs_icp_band').on(t.icpBand), index('idx_orgs_state').on(t.state), ], diff --git a/packages/outreach-core/src/orgs/actions/upsert-org-from-registry.server.ts b/packages/outreach-core/src/orgs/actions/upsert-org-from-registry.server.ts index 0386817..c5de70d 100644 --- a/packages/outreach-core/src/orgs/actions/upsert-org-from-registry.server.ts +++ b/packages/outreach-core/src/orgs/actions/upsert-org-from-registry.server.ts @@ -6,8 +6,10 @@ import { schema } from '#~/db/db.js'; export interface RegistryOrgInput { readonly name: string; readonly city: string | null; - readonly state?: string; - readonly registrationStatus: 'good_standing' | 'lapsed' | 'unknown'; + readonly state?: string | null; + /** Registry-assigned id (e.g. NHDOJ Reg. No.) — preferred identity when present. */ + readonly registrationNumber?: string | null; + readonly registrationStatus: 'good_standing' | 'lapsed' | 'suspended' | 'unknown'; readonly sourceRegistry: string; } @@ -18,12 +20,17 @@ export interface RegistryUpsertResult { /** * Upserts an org discovered in a charity registry re-scan (e.g. the NHDOJ - * Charitable Trusts list). Registries carry no EIN, so identity is the - * case-insensitive (name, city, state) triple — good enough for a - * single-state registry where the same legal name in the same city is the - * same org. Existing rows get their registration status refreshed (orgs - * lapse and re-register); enrichment fields (EIN, revenue, NTEE) are never - * touched here. + * Charitable Trusts list). + * + * Identity, in preference order: + * 1. (sourceRegistry, registrationNumber) — the registry's own stable id; + * survives renames and relocations. Name/city/state are refreshed from + * the incoming row on this path. + * 2. Case-insensitive (name, city, state) — fallback for registries (or + * historical rows) without a registration number. Only status and + * provenance are refreshed on this path. + * + * Enrichment fields (EIN, revenue, NTEE) are never touched here. * * Returns `inserted: true` for brand-new registrants — a segment the * pipeline treats specially (often actively seeking first-time funding). @@ -33,8 +40,37 @@ export async function serverUpsertOrgFromRegistry( org: RegistryOrgInput, ): Promise { const state = org.state ?? 'NH'; + const registrationNumber = org.registrationNumber ?? null; - const existing = await db + if (registrationNumber != null) { + const byRegNo = await db + .select({ id: schema.orgs.id }) + .from(schema.orgs) + .where( + and( + eq(schema.orgs.sourceRegistry, org.sourceRegistry), + eq(schema.orgs.registrationNumber, registrationNumber), + ), + ) + .limit(1); + + const found = byRegNo[0]; + if (found != null) { + await db + .update(schema.orgs) + .set({ + name: org.name, + city: org.city, + state, + registrationStatus: org.registrationStatus, + updatedAt: sql`now()`, + }) + .where(eq(schema.orgs.id, found.id)); + return { id: found.id, inserted: false }; + } + } + + const byName = await db .select({ id: schema.orgs.id }) .from(schema.orgs) .where( @@ -48,13 +84,15 @@ export async function serverUpsertOrgFromRegistry( ) .limit(1); - const found = existing[0]; + const found = byName[0]; if (found != null) { await db .update(schema.orgs) .set({ registrationStatus: org.registrationStatus, sourceRegistry: org.sourceRegistry, + // Adopt the registry id so future re-scans match on the stable key. + registrationNumber, updatedAt: sql`now()`, }) .where(eq(schema.orgs.id, found.id)); @@ -67,6 +105,7 @@ export async function serverUpsertOrgFromRegistry( name: org.name, city: org.city, state, + registrationNumber, registrationStatus: org.registrationStatus, sourceRegistry: org.sourceRegistry, }) diff --git a/packages/outreach-core/src/orgs/queries/list-orgs-needing-enrichment.server.ts b/packages/outreach-core/src/orgs/queries/list-orgs-needing-enrichment.server.ts index 32460ee..39f0e61 100644 --- a/packages/outreach-core/src/orgs/queries/list-orgs-needing-enrichment.server.ts +++ b/packages/outreach-core/src/orgs/queries/list-orgs-needing-enrichment.server.ts @@ -1,4 +1,4 @@ -import { and, asc, isNull } from 'drizzle-orm'; +import { and, asc, isNull, sql } from 'drizzle-orm'; import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; import { schema } from '#~/db/db.js'; @@ -31,6 +31,12 @@ export async function serverListOrgsNeedingEnrichment( }) .from(schema.orgs) .where(and(isNull(schema.orgs.ein), isNull(schema.orgs.totalRevenue))) - .orderBy(asc(schema.orgs.updatedAt)) + // NH good-standing orgs are the ICP — enrich them before the long tail + // of out-of-state registrants; oldest-touched first within each tier. + .orderBy( + sql`(${schema.orgs.state} = 'NH') DESC`, + sql`(${schema.orgs.registrationStatus} = 'good_standing') DESC`, + asc(schema.orgs.updatedAt), + ) .limit(limit); }