feat(ingestion): Phase 1 data spine — Grants.gov, PND RSS, NHDOJ registry, ProPublica enrichment

Core: icpBandForRevenue (100K-5M primary), serverUpsertOrgFromRegistry
(case-insensitive name/city/state key, new-registrant signal),
serverEnrichOrg (IRS fields + re-band, all-null marks attempted),
serverListOrgsNeedingEnrichment.

Worker: four source verticals, each a thin fail-loud client + pure
tested normalize layer + DBOS scheduled workflow:
- ingestGrants (nightly): Search2 paginated (cap 1000) -> fetchOpportunity
  details (cap 200, logged drops, 250ms politeness) -> batched upsert
- ingestPndRss (nightly): RSS via fast-xml-parser, heuristic funder/
  deadline extraction, link-keyed upsert
- ingestNhdojOrgs (monthly): pdfjs-dist positioned-text extraction, pure
  row reconstruction (multi-line names, inferred columns, fail-loud on
  layout change), registry upsert; no-op warn when PDF URL unset
- enrichOrgs (daily): ProPublica search -> conservative name/city match
  (null beats guess) -> latest-filing revenue/NTEE/FYE, per-org steps for
  checkpointed resume, >20% batch failure rethrows

94 worker + 16 core + 5 ai tests green; docs/features/ingestion.md added.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/outreach-worker/src/sources/grants-gov/normalize.ts

@@ -0,0 +1,181 @@
+/**
+ * Pure normalization from Grants.gov's wire shapes to our `grants` schema.
+ * No fetch, no I/O — everything here is a deterministic function of its
+ * inputs so it can be unit tested against fixture JSON without a network.
+ */
+import type { NewGrantInput } from '@novelpad/outreach-core/server';
+
+import type {
+  GrantsGovApplicantType,
+  GrantsGovOpportunityDetail,
+  GrantsGovSearchHit,
+} from '#~/sources/grants-gov/client.js';
+
+const GRANTS_GOV_DATE_RE = /^(\d{2})\/(\d{2})\/(\d{4})$/;
+
+/**
+ * Parses a Grants.gov date. The typical wire format is `MM/DD/YYYY`; a
+ * handful of fields have historically arrived as ISO-ish strings, so that's
+ * accepted as a fallback. Anything that doesn't parse to a real calendar
+ * date — including out-of-range components and month/day rollover (e.g.
+ * "02/30/2024") — returns `null` rather than a garbage `Date`.
+ */
+export function parseGrantsGovDate(
+  value: string | null | undefined,
+): Date | null {
+  if (value == null) return null;
+  const trimmed = value.trim();
+  if (trimmed === '') return null;
+
+  const match = GRANTS_GOV_DATE_RE.exec(trimmed);
+  if (match != null) {
+    const month = Number(match[1]);
+    const day = Number(match[2]);
+    const year = Number(match[3]);
+    if (month < 1 || month > 12 || day < 1 || day > 31) return null;
+
+    const date = new Date(Date.UTC(year, month - 1, day));
+    const rolledOver =
+      date.getUTCFullYear() !== year ||
+      date.getUTCMonth() !== month - 1 ||
+      date.getUTCDate() !== day;
+    return rolledOver ? null : date;
+  }
+
+  const fallback = new Date(trimmed);
+  return Number.isNaN(fallback.getTime()) ? null : fallback;
+}
+
+/**
+ * Parses an award amount that may arrive as a number or as a formatted
+ * string (`"$1,500,000"`, `"1500000"`). Schema stores whole dollars — award
+ * amounts are never sub-dollar, so the result is rounded.
+ */
+export function parseDollarAmount(
+  value: string | number | null | undefined,
+): number | null {
+  if (value == null) return null;
+  if (typeof value === 'number') {
+    return Number.isFinite(value) ? Math.round(value) : null;
+  }
+  const cleaned = value.replace(/[$,]/g, '').trim();
+  if (cleaned === '') return null;
+  const parsed = Number(cleaned);
+  return Number.isFinite(parsed) ? Math.round(parsed) : null;
+}
+
+/** Same coercion rules as `parseDollarAmount`, for plain counts (not money). */
+export function parseCount(
+  value: string | number | null | undefined,
+): number | null {
+  return parseDollarAmount(value);
+}
+
+/**
+ * Strips HTML tags from a synopsis description, converting common block
+ * boundaries to newlines and decoding the handful of entities Grants.gov
+ * actually emits. Not a general-purpose HTML sanitizer — synopses are
+ * federal agency copy, not untrusted user content.
+ */
+export function stripHtml(html: string | null | undefined): string | null {
+  if (html == null) return null;
+
+  const text = html
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/(p|div|li)>/gi, '\n')
+    .replace(/<[^>]+>/g, '')
+    .replace(/&nbsp;/gi, ' ')
+    .replace(/&amp;/gi, '&')
+    .replace(/&lt;/gi, '<')
+    .replace(/&gt;/gi, '>')
+    .replace(/&#39;/g, "'")
+    .replace(/&quot;/g, '"')
+    .replace(/\r\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+
+  return text === '' ? null : text;
+}
+
+/** Extracts non-empty eligibility descriptions from `synopsis.applicantTypes`. */
+export function normalizeApplicantTypes(
+  applicantTypes: ReadonlyArray<GrantsGovApplicantType> | null | undefined,
+): string[] | null {
+  if (applicantTypes == null || applicantTypes.length === 0) return null;
+
+  const descriptions = applicantTypes
+    .map((type) => type.description?.trim())
+    .filter((description): description is string => Boolean(description));
+
+  return descriptions.length === 0 ? null : descriptions;
+}
+
+function firstNonEmpty(
+  ...values: ReadonlyArray<string | null | undefined>
+): string | null {
+  for (const value of values) {
+    const trimmed = value?.trim();
+    if (trimmed) return trimmed;
+  }
+  return null;
+}
+
+/**
+ * Resolves the funder display name: prefer the detail response's agency
+ * name (most complete), fall back to the search hit's agency name, then
+ * either response's agency code, then a literal placeholder so we never
+ * write an empty `funder`.
+ */
+function resolveFunder(
+  hit: GrantsGovSearchHit,
+  detail: GrantsGovOpportunityDetail,
+): string {
+  return (
+    firstNonEmpty(
+      detail.agencyDetails?.agencyName,
+      detail.synopsis?.agencyName,
+      hit.agency,
+    ) ??
+    firstNonEmpty(detail.agencyDetails?.agencyCode, hit.agencyCode) ??
+    'Unknown federal agency'
+  );
+}
+
+/**
+ * Combines a `search2` hit with its `fetchOpportunity` detail into an
+ * insertable grant row. `sourceUrl` is the schema's upsert key, so it's
+ * derived purely from the opportunity id and stable across re-crawls.
+ *
+ * `geographicScope` is deliberately left `null`: federal opportunities are
+ * national by default and Grants.gov doesn't give us a reliable field to
+ * derive a narrower scope from — better to leave it unset than guess.
+ */
+export function normalizeGrantsGovOpportunity(
+  hit: GrantsGovSearchHit,
+  detail: GrantsGovOpportunityDetail,
+): NewGrantInput {
+  const synopsis = detail.synopsis;
+
+  return {
+    sourceUrl: `https://www.grants.gov/search-results-detail/${hit.id}`,
+    source: 'grants_gov',
+    title: detail.opportunityTitle ?? hit.title,
+    funder: resolveFunder(hit, detail),
+    synopsis: stripHtml(synopsis?.synopsisDesc),
+    eligibilityEntityTypes: normalizeApplicantTypes(
+      synopsis?.applicantTypes,
+    ),
+    geographicScope: null,
+    programAreas: null,
+    awardFloor: parseDollarAmount(synopsis?.awardFloor),
+    awardCeiling: parseDollarAmount(synopsis?.awardCeiling),
+    expectedAwardsCount: parseCount(synopsis?.expectedNumberOfAwards),
+    openDate: parseGrantsGovDate(hit.openDate ?? synopsis?.postingDate),
+    closeDate: parseGrantsGovDate(hit.closeDate ?? synopsis?.responseDate),
+    matchRequirement: false,
+    applicationEffortEstimate: 'unknown',
+    applicationFormSupported: false,
+    status: 'open',
+    lastVerifiedAt: new Date(),
+  };
+}