feat(ingestion): Phase 1 data spine — Grants.gov, PND RSS, NHDOJ registry, ProPublica enrichment

Core: icpBandForRevenue (100K-5M primary), serverUpsertOrgFromRegistry
(case-insensitive name/city/state key, new-registrant signal),
serverEnrichOrg (IRS fields + re-band, all-null marks attempted),
serverListOrgsNeedingEnrichment.

Worker: four source verticals, each a thin fail-loud client + pure
tested normalize layer + DBOS scheduled workflow:
- ingestGrants (nightly): Search2 paginated (cap 1000) -> fetchOpportunity
  details (cap 200, logged drops, 250ms politeness) -> batched upsert
- ingestPndRss (nightly): RSS via fast-xml-parser, heuristic funder/
  deadline extraction, link-keyed upsert
- ingestNhdojOrgs (monthly): pdfjs-dist positioned-text extraction, pure
  row reconstruction (multi-line names, inferred columns, fail-loud on
  layout change), registry upsert; no-op warn when PDF URL unset
- enrichOrgs (daily): ProPublica search -> conservative name/city match
  (null beats guess) -> latest-filing revenue/NTEE/FYE, per-org steps for
  checkpointed resume, >20% batch failure rethrows

94 worker + 16 core + 5 ai tests green; docs/features/ingestion.md added.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/outreach-worker/src/workflows/ingest-nhdoj-orgs.ts

@@ -0,0 +1,141 @@
+/**
+ * Monthly NHDOJ Charitable Trusts registry ingestion workflow.
+ *
+ * The NH Department of Justice Charitable Trusts Unit publishes a PDF
+ * roster of registered charitable organizations. Unlike Grants.gov/PND/
+ * ProPublica this isn't a grants feed — it's an org registry re-scan, so a
+ * fresh registrant (`inserted: true` from `serverUpsertOrgFromRegistry`) is
+ * itself the signal: it's a segment often actively seeking first-time
+ * funding.
+ *
+ * Registration follows `ingest-grants.ts` exactly: the scheduled function
+ * must ALSO be registered as a plain workflow (both registrations
+ * referencing the same function object), and deps are pulled from a
+ * module-scope registry populated before `DBOS.launch()` — DBOS serializes
+ * workflow args, so closures/functions can't cross that boundary.
+ */
+import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
+import type { schema } from '@novelpad/outreach-core';
+import { serverUpsertOrgFromRegistry } from '@novelpad/outreach-core/server';
+import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
+
+import { extractPositionedText } from '#~/sources/nhdoj/extract-pdf-text.js';
+import {
+  normalizeRegistryRows,
+  reconstructRegistryRows,
+  type NormalizedRegistryRow,
+} from '#~/sources/nhdoj/parse-registry.js';
+
+export type OutreachDb = NodePgDatabase<typeof schema>;
+
+const SOURCE_REGISTRY = 'nhdoj_charitable_trusts';
+
+export interface IngestNhdojOrgsDeps {
+  readonly db: OutreachDb;
+}
+
+let registeredDeps: IngestNhdojOrgsDeps | null = null;
+
+export function setIngestNhdojOrgsDeps(deps: IngestNhdojOrgsDeps): void {
+  registeredDeps = deps;
+}
+
+function getIngestNhdojOrgsDeps(): IngestNhdojOrgsDeps {
+  if (registeredDeps == null) {
+    throw new Error(
+      'IngestNhdojOrgsDeps not registered. Call setIngestNhdojOrgsDeps() before DBOS.launch().',
+    );
+  }
+  return registeredDeps;
+}
+
+async function fetchNhdojRegistryPdf(url: string): Promise<Uint8Array> {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(
+      `fetchNhdojRegistryPdf: ${url} responded ${res.status} ${res.statusText}`,
+    );
+  }
+  const buf = await res.arrayBuffer();
+  return new Uint8Array(buf);
+}
+const fetchNhdojRegistryPdfStep = DBOS.registerStep(fetchNhdojRegistryPdf, {
+  name: 'fetchNhdojRegistryPdf',
+  retriesAllowed: true,
+  maxAttempts: 3,
+});
+
+async function upsertRegistryOrg(
+  db: OutreachDb,
+  row: NormalizedRegistryRow,
+): Promise<{ id: string; inserted: boolean }> {
+  return serverUpsertOrgFromRegistry(db, {
+    name: row.name,
+    city: row.city,
+    registrationStatus: row.status,
+    sourceRegistry: SOURCE_REGISTRY,
+  });
+}
+const upsertRegistryOrgStep = DBOS.registerStep(upsertRegistryOrg, {
+  name: 'upsertNhdojRegistryOrg',
+  retriesAllowed: true,
+  maxAttempts: 3,
+});
+
+async function runIngestNhdojOrgs(): Promise<void> {
+  const { db } = getIngestNhdojOrgsDeps();
+
+  const pdfUrl = process.env.NHDOJ_REGISTRY_PDF_URL;
+  if (pdfUrl == null || pdfUrl.trim() === '') {
+    // The registry PDF's URL changes whenever NHDOJ republishes it (no
+    // stable endpoint) — a hard failure here would page on-call for a
+    // config gap rather than a real outage, so skip quietly instead.
+    console.warn(
+      '[ingest-nhdoj-orgs] NHDOJ_REGISTRY_PDF_URL is not set; skipping this pass.',
+    );
+    return;
+  }
+
+  const pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl);
+  const pages = await extractPositionedText(pdfBytes);
+  const rawRows = reconstructRegistryRows(pages);
+  const normalizedRows = normalizeRegistryRows(rawRows);
+
+  let upserted = 0;
+  let newlyInserted = 0;
+  for (const row of normalizedRows) {
+    const result = await upsertRegistryOrgStep(db, row);
+    upserted++;
+    if (result.inserted) newlyInserted++;
+  }
+
+  console.log(
+    `[ingest-nhdoj-orgs] rowsParsed=${normalizedRows.length} upserted=${upserted} newlyInserted=${newlyInserted}`,
+  );
+}
+
+const g = globalThis as unknown as {
+  __outreachIngestNhdojOrgsRegistered?: boolean;
+};
+
+if (!g.__outreachIngestNhdojOrgsRegistered) {
+  g.__outreachIngestNhdojOrgsRegistered = true;
+
+  const ingestNhdojOrgs = async (_scheduledTime: Date, _startedAt: Date) => {
+    try {
+      await runIngestNhdojOrgs();
+    } catch (err) {
+      console.error('[ingest-nhdoj-orgs] pass failed:', err);
+      throw err;
+    }
+  };
+
+  // Must be registered as BOTH a workflow and a scheduled function,
+  // referencing the same function object — see module doc comment.
+  DBOS.registerWorkflow(ingestNhdojOrgs, { name: 'ingestNhdojOrgs' });
+  DBOS.registerScheduled(ingestNhdojOrgs, {
+    crontab: '0 4 1 * *',
+    name: 'ingestNhdojOrgs',
+    mode: SchedulerMode.ExactlyOncePerInterval,
+  });
+}