grant-outreach-engine/apps/outreach-worker/src/workflows/ingest-nhdoj-orgs.ts

/**
 * Monthly NHDOJ Charitable Trusts registry ingestion workflow.
 *
 * The NH Department of Justice Charitable Trusts Unit publishes a PDF
 * roster of registered charitable organizations. Unlike Grants.gov/PND/
 * ProPublica this isn't a grants feed — it's an org registry re-scan, so a
 * fresh registrant (`inserted: true` from `serverUpsertOrgFromRegistry`) is
 * itself the signal: it's a segment often actively seeking first-time
 * funding.
 *
 * Registration follows `ingest-grants.ts` exactly: the scheduled function
 * must ALSO be registered as a plain workflow (both registrations
 * referencing the same function object), and deps are pulled from a
 * module-scope registry populated before `DBOS.launch()` — DBOS serializes
 * workflow args, so closures/functions can't cross that boundary.
 */
import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
import type { schema } from '@novelpad/outreach-core';
import { serverUpsertOrgFromRegistry } from '@novelpad/outreach-core/server';
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';

import { extractPositionedText } from '#~/sources/nhdoj/extract-pdf-text.js';
import {
  normalizeRegistryRows,
  reconstructRegistryRows,
  type NormalizedRegistryRow,
} from '#~/sources/nhdoj/parse-registry.js';

export type OutreachDb = NodePgDatabase<typeof schema>;

const SOURCE_REGISTRY = 'nhdoj_charitable_trusts';

export interface IngestNhdojOrgsDeps {
  readonly db: OutreachDb;
}

let registeredDeps: IngestNhdojOrgsDeps | null = null;

export function setIngestNhdojOrgsDeps(deps: IngestNhdojOrgsDeps): void {
  registeredDeps = deps;
}

function getIngestNhdojOrgsDeps(): IngestNhdojOrgsDeps {
  if (registeredDeps == null) {
    throw new Error(
      'IngestNhdojOrgsDeps not registered. Call setIngestNhdojOrgsDeps() before DBOS.launch().',
    );
  }
  return registeredDeps;
}

async function fetchNhdojRegistryPdf(url: string): Promise<Uint8Array> {
  const res = await fetch(url);
  if (!res.ok) {
    throw new Error(
      `fetchNhdojRegistryPdf: ${url} responded ${res.status} ${res.statusText}`,
    );
  }
  const buf = await res.arrayBuffer();
  return new Uint8Array(buf);
}
const fetchNhdojRegistryPdfStep = DBOS.registerStep(fetchNhdojRegistryPdf, {
  name: 'fetchNhdojRegistryPdf',
  retriesAllowed: true,
  maxAttempts: 3,
});

async function upsertRegistryOrg(
  db: OutreachDb,
  row: NormalizedRegistryRow,
): Promise<{ id: string; inserted: boolean }> {
  return serverUpsertOrgFromRegistry(db, {
    name: row.name,
    city: row.city,
    registrationStatus: row.status,
    sourceRegistry: SOURCE_REGISTRY,
  });
}
const upsertRegistryOrgStep = DBOS.registerStep(upsertRegistryOrg, {
  name: 'upsertNhdojRegistryOrg',
  retriesAllowed: true,
  maxAttempts: 3,
});

async function runIngestNhdojOrgs(): Promise<void> {
  const { db } = getIngestNhdojOrgsDeps();

  const pdfUrl = process.env.NHDOJ_REGISTRY_PDF_URL;
  if (pdfUrl == null || pdfUrl.trim() === '') {
    // The registry PDF's URL changes whenever NHDOJ republishes it (no
    // stable endpoint) — a hard failure here would page on-call for a
    // config gap rather than a real outage, so skip quietly instead.
    console.warn(
      '[ingest-nhdoj-orgs] NHDOJ_REGISTRY_PDF_URL is not set; skipping this pass.',
    );
    return;
  }

  const pdfBytes = await fetchNhdojRegistryPdfStep(pdfUrl);
  const pages = await extractPositionedText(pdfBytes);
  const rawRows = reconstructRegistryRows(pages);
  const normalizedRows = normalizeRegistryRows(rawRows);

  let upserted = 0;
  let newlyInserted = 0;
  for (const row of normalizedRows) {
    const result = await upsertRegistryOrgStep(db, row);
    upserted++;
    if (result.inserted) newlyInserted++;
  }

  console.log(
    `[ingest-nhdoj-orgs] rowsParsed=${normalizedRows.length} upserted=${upserted} newlyInserted=${newlyInserted}`,
  );
}

const g = globalThis as unknown as {
  __outreachIngestNhdojOrgsRegistered?: boolean;
};

if (!g.__outreachIngestNhdojOrgsRegistered) {
  g.__outreachIngestNhdojOrgsRegistered = true;

  const ingestNhdojOrgs = async (_scheduledTime: Date, _startedAt: Date) => {
    try {
      await runIngestNhdojOrgs();
    } catch (err) {
      console.error('[ingest-nhdoj-orgs] pass failed:', err);
      throw err;
    }
  };

  // Must be registered as BOTH a workflow and a scheduled function,
  // referencing the same function object — see module doc comment.
  DBOS.registerWorkflow(ingestNhdojOrgs, { name: 'ingestNhdojOrgs' });
  DBOS.registerScheduled(ingestNhdojOrgs, {
    crontab: '0 4 1 * *',
    name: 'ingestNhdojOrgs',
    mode: SchedulerMode.ExactlyOncePerInterval,
  });
}