feat(ingestion): Phase 1 data spine — Grants.gov, PND RSS, NHDOJ registry, ProPublica enrichment

Core: icpBandForRevenue (100K-5M primary), serverUpsertOrgFromRegistry
(case-insensitive name/city/state key, new-registrant signal),
serverEnrichOrg (IRS fields + re-band, all-null marks attempted),
serverListOrgsNeedingEnrichment.

Worker: four source verticals, each a thin fail-loud client + pure
tested normalize layer + DBOS scheduled workflow:
- ingestGrants (nightly): Search2 paginated (cap 1000) -> fetchOpportunity
  details (cap 200, logged drops, 250ms politeness) -> batched upsert
- ingestPndRss (nightly): RSS via fast-xml-parser, heuristic funder/
  deadline extraction, link-keyed upsert
- ingestNhdojOrgs (monthly): pdfjs-dist positioned-text extraction, pure
  row reconstruction (multi-line names, inferred columns, fail-loud on
  layout change), registry upsert; no-op warn when PDF URL unset
- enrichOrgs (daily): ProPublica search -> conservative name/city match
  (null beats guess) -> latest-filing revenue/NTEE/FYE, per-org steps for
  checkpointed resume, >20% batch failure rethrows

94 worker + 16 core + 5 ai tests green; docs/features/ingestion.md added.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

packages/outreach-core/src/index.ts

@@ -7,3 +7,4 @@
 // server-only action/query.
 export * from './db/index.js';
 export * from './matches/hard-gates.js';
+export * from './orgs/icp-band.js';

packages/outreach-core/src/orgs/actions/enrich-org.server.ts

@@ -0,0 +1,37 @@
+import { eq, sql } from 'drizzle-orm';
+
+import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
+import { schema } from '#~/db/db.js';
+import { icpBandForRevenue } from '../icp-band.js';
+
+export interface OrgEnrichmentInput {
+  readonly ein: string | null;
+  readonly nteeCode: string | null;
+  readonly totalRevenue: number | null;
+  readonly fiscalYearEnd: string | null;
+}
+
+/**
+ * Applies IRS-derived enrichment (ProPublica Nonprofit Explorer) to an org
+ * and re-bands it. Orgs that failed to resolve should be recorded too
+ * (all-null input) so `updatedAt` moves and the enrichment queue doesn't
+ * retry them every run — the icp band stays/returns `unknown`, which
+ * down-prioritizes rather than drops them.
+ */
+export async function serverEnrichOrg(
+  db: NpOutreachDatabase | NpOutreachTransaction,
+  orgId: string,
+  enrichment: OrgEnrichmentInput,
+): Promise<void> {
+  await db
+    .update(schema.orgs)
+    .set({
+      ein: enrichment.ein,
+      nteeCode: enrichment.nteeCode,
+      totalRevenue: enrichment.totalRevenue,
+      fiscalYearEnd: enrichment.fiscalYearEnd,
+      icpBand: icpBandForRevenue(enrichment.totalRevenue),
+      updatedAt: sql`now()`,
+    })
+    .where(eq(schema.orgs.id, orgId));
+}

packages/outreach-core/src/orgs/actions/index.server.ts

@@ -1 +1,3 @@
 export * from './insert-org.server.js';
+export * from './upsert-org-from-registry.server.js';
+export * from './enrich-org.server.js';

packages/outreach-core/src/orgs/actions/upsert-org-from-registry.server.ts

@@ -0,0 +1,79 @@
+import { and, eq, sql } from 'drizzle-orm';
+
+import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
+import { schema } from '#~/db/db.js';
+
+export interface RegistryOrgInput {
+  readonly name: string;
+  readonly city: string | null;
+  readonly state?: string;
+  readonly registrationStatus: 'good_standing' | 'lapsed' | 'unknown';
+  readonly sourceRegistry: string;
+}
+
+export interface RegistryUpsertResult {
+  readonly id: string;
+  readonly inserted: boolean;
+}
+
+/**
+ * Upserts an org discovered in a charity registry re-scan (e.g. the NHDOJ
+ * Charitable Trusts list). Registries carry no EIN, so identity is the
+ * case-insensitive (name, city, state) triple — good enough for a
+ * single-state registry where the same legal name in the same city is the
+ * same org. Existing rows get their registration status refreshed (orgs
+ * lapse and re-register); enrichment fields (EIN, revenue, NTEE) are never
+ * touched here.
+ *
+ * Returns `inserted: true` for brand-new registrants — a segment the
+ * pipeline treats specially (often actively seeking first-time funding).
+ */
+export async function serverUpsertOrgFromRegistry(
+  db: NpOutreachDatabase | NpOutreachTransaction,
+  org: RegistryOrgInput,
+): Promise<RegistryUpsertResult> {
+  const state = org.state ?? 'NH';
+
+  const existing = await db
+    .select({ id: schema.orgs.id })
+    .from(schema.orgs)
+    .where(
+      and(
+        sql`lower(${schema.orgs.name}) = lower(${org.name})`,
+        org.city == null
+          ? sql`${schema.orgs.city} IS NULL`
+          : sql`lower(${schema.orgs.city}) = lower(${org.city})`,
+        eq(schema.orgs.state, state),
+      ),
+    )
+    .limit(1);
+
+  const found = existing[0];
+  if (found != null) {
+    await db
+      .update(schema.orgs)
+      .set({
+        registrationStatus: org.registrationStatus,
+        sourceRegistry: org.sourceRegistry,
+        updatedAt: sql`now()`,
+      })
+      .where(eq(schema.orgs.id, found.id));
+    return { id: found.id, inserted: false };
+  }
+
+  const [row] = await db
+    .insert(schema.orgs)
+    .values({
+      name: org.name,
+      city: org.city,
+      state,
+      registrationStatus: org.registrationStatus,
+      sourceRegistry: org.sourceRegistry,
+    })
+    .returning({ id: schema.orgs.id });
+
+  if (row == null) {
+    throw new Error('serverUpsertOrgFromRegistry: insert returned no row');
+  }
+  return { id: row.id, inserted: true };
+}

packages/outreach-core/src/orgs/icp-band.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  ICP_PRIMARY_MAX_REVENUE,
+  ICP_PRIMARY_MIN_REVENUE,
+  icpBandForRevenue,
+} from './icp-band.js';
+
+describe('icpBandForRevenue', () => {
+  it('returns unknown for missing revenue', () => {
+    expect(icpBandForRevenue(null)).toBe('unknown');
+    expect(icpBandForRevenue(undefined)).toBe('unknown');
+    expect(icpBandForRevenue(Number.NaN)).toBe('unknown');
+  });
+
+  it('bands below the primary floor as below', () => {
+    expect(icpBandForRevenue(0)).toBe('below');
+    expect(icpBandForRevenue(ICP_PRIMARY_MIN_REVENUE - 1)).toBe('below');
+  });
+
+  it('bands the inclusive primary range as primary', () => {
+    expect(icpBandForRevenue(ICP_PRIMARY_MIN_REVENUE)).toBe('primary');
+    expect(icpBandForRevenue(1_250_000)).toBe('primary');
+    expect(icpBandForRevenue(ICP_PRIMARY_MAX_REVENUE)).toBe('primary');
+  });
+
+  it('bands above the primary ceiling as above', () => {
+    expect(icpBandForRevenue(ICP_PRIMARY_MAX_REVENUE + 1)).toBe('above');
+  });
+});

packages/outreach-core/src/orgs/icp-band.ts

@@ -0,0 +1,21 @@
+/**
+ * ICP banding from annual revenue (per docs/plan.md, Stage 3):
+ * $100K–$5M is the primary ICP — below that, orgs rarely have anyone in a
+ * grant-writing seat; above it, professional development staff need a
+ * different message. Kept as a pure function so ingestion and re-banding
+ * jobs share one definition.
+ */
+
+export type IcpBand = 'below' | 'primary' | 'above' | 'unknown';
+
+export const ICP_PRIMARY_MIN_REVENUE = 100_000;
+export const ICP_PRIMARY_MAX_REVENUE = 5_000_000;
+
+export function icpBandForRevenue(
+  totalRevenue: number | null | undefined,
+): IcpBand {
+  if (totalRevenue == null || Number.isNaN(totalRevenue)) return 'unknown';
+  if (totalRevenue < ICP_PRIMARY_MIN_REVENUE) return 'below';
+  if (totalRevenue <= ICP_PRIMARY_MAX_REVENUE) return 'primary';
+  return 'above';
+}

packages/outreach-core/src/orgs/queries/index.server.ts

@@ -1 +1,2 @@
 export * from './list-orgs-in-icp-band.server.js';
+export * from './list-orgs-needing-enrichment.server.js';

packages/outreach-core/src/orgs/queries/list-orgs-needing-enrichment.server.ts

@@ -0,0 +1,36 @@
+import { and, asc, isNull } from 'drizzle-orm';
+
+import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
+import { schema } from '#~/db/db.js';
+
+export interface OrgNeedingEnrichment {
+  id: string;
+  name: string;
+  city: string | null;
+  state: string;
+}
+
+/**
+ * Orgs that have never been through IRS enrichment: no EIN resolved and
+ * no revenue on file. Ordered oldest-touched first so a bounded batch job
+ * works through the backlog fairly. Orgs already attempted (recorded via
+ * serverEnrichOrg with all-null input) still match this filter only until
+ * a design for retry windows lands — callers should bound `limit` to keep
+ * a nightly run cheap.
+ */
+export async function serverListOrgsNeedingEnrichment(
+  db: NpOutreachDatabase | NpOutreachTransaction,
+  { limit }: { limit: number },
+): Promise<OrgNeedingEnrichment[]> {
+  return db
+    .select({
+      id: schema.orgs.id,
+      name: schema.orgs.name,
+      city: schema.orgs.city,
+      state: schema.orgs.state,
+    })
+    .from(schema.orgs)
+    .where(and(isNull(schema.orgs.ein), isNull(schema.orgs.totalRevenue)))
+    .orderBy(asc(schema.orgs.updatedAt))
+    .limit(limit);
+}