grant-outreach-engine/apps/outreach-worker/src/sources/grants-gov/normalize.ts

/**
 * Pure normalization from Grants.gov's wire shapes to our `grants` schema.
 * No fetch, no I/O — everything here is a deterministic function of its
 * inputs so it can be unit tested against fixture JSON without a network.
 */
import type { NewGrantInput } from '@novelpad/outreach-core/server';

import type {
  GrantsGovApplicantType,
  GrantsGovOpportunityDetail,
  GrantsGovSearchHit,
} from '#~/sources/grants-gov/client.js';

const GRANTS_GOV_DATE_RE = /^(\d{2})\/(\d{2})\/(\d{4})$/;

/**
 * Parses a Grants.gov date. The typical wire format is `MM/DD/YYYY`; a
 * handful of fields have historically arrived as ISO-ish strings, so that's
 * accepted as a fallback. Anything that doesn't parse to a real calendar
 * date — including out-of-range components and month/day rollover (e.g.
 * "02/30/2024") — returns `null` rather than a garbage `Date`.
 */
export function parseGrantsGovDate(
  value: string | null | undefined,
): Date | null {
  if (value == null) return null;
  const trimmed = value.trim();
  if (trimmed === '') return null;

  const match = GRANTS_GOV_DATE_RE.exec(trimmed);
  if (match != null) {
    const month = Number(match[1]);
    const day = Number(match[2]);
    const year = Number(match[3]);
    if (month < 1 || month > 12 || day < 1 || day > 31) return null;

    const date = new Date(Date.UTC(year, month - 1, day));
    const rolledOver =
      date.getUTCFullYear() !== year ||
      date.getUTCMonth() !== month - 1 ||
      date.getUTCDate() !== day;
    return rolledOver ? null : date;
  }

  const fallback = new Date(trimmed);
  return Number.isNaN(fallback.getTime()) ? null : fallback;
}

/**
 * Parses an award amount that may arrive as a number or as a formatted
 * string (`"$1,500,000"`, `"1500000"`). Schema stores whole dollars — award
 * amounts are never sub-dollar, so the result is rounded.
 */
export function parseDollarAmount(
  value: string | number | null | undefined,
): number | null {
  if (value == null) return null;
  if (typeof value === 'number') {
    return Number.isFinite(value) ? Math.round(value) : null;
  }
  const cleaned = value.replace(/[$,]/g, '').trim();
  if (cleaned === '') return null;
  const parsed = Number(cleaned);
  return Number.isFinite(parsed) ? Math.round(parsed) : null;
}

/** Same coercion rules as `parseDollarAmount`, for plain counts (not money). */
export function parseCount(
  value: string | number | null | undefined,
): number | null {
  return parseDollarAmount(value);
}

/**
 * Strips HTML tags from a synopsis description, converting common block
 * boundaries to newlines and decoding the handful of entities Grants.gov
 * actually emits. Not a general-purpose HTML sanitizer — synopses are
 * federal agency copy, not untrusted user content.
 */
export function stripHtml(html: string | null | undefined): string | null {
  if (html == null) return null;

  const text = html
    .replace(/<br\s*\/?>/gi, '\n')
    .replace(/<\/(p|div|li)>/gi, '\n')
    .replace(/<[^>]+>/g, '')
    .replace(/&nbsp;/gi, ' ')
    .replace(/&amp;/gi, '&')
    .replace(/&lt;/gi, '<')
    .replace(/&gt;/gi, '>')
    .replace(/&#39;/g, "'")
    .replace(/&quot;/g, '"')
    .replace(/\r\n/g, '\n')
    .replace(/\n{3,}/g, '\n\n')
    .trim();

  return text === '' ? null : text;
}

/** Extracts non-empty eligibility descriptions from `synopsis.applicantTypes`. */
export function normalizeApplicantTypes(
  applicantTypes: ReadonlyArray<GrantsGovApplicantType> | null | undefined,
): string[] | null {
  if (applicantTypes == null || applicantTypes.length === 0) return null;

  const descriptions = applicantTypes
    .map((type) => type.description?.trim())
    .filter((description): description is string => Boolean(description));

  return descriptions.length === 0 ? null : descriptions;
}

function firstNonEmpty(
  ...values: ReadonlyArray<string | null | undefined>
): string | null {
  for (const value of values) {
    const trimmed = value?.trim();
    if (trimmed) return trimmed;
  }
  return null;
}

/**
 * Resolves the funder display name: prefer the detail response's agency
 * name (most complete), fall back to the search hit's agency name, then
 * either response's agency code, then a literal placeholder so we never
 * write an empty `funder`.
 */
function resolveFunder(
  hit: GrantsGovSearchHit,
  detail: GrantsGovOpportunityDetail,
): string {
  return (
    firstNonEmpty(
      detail.agencyDetails?.agencyName,
      detail.synopsis?.agencyName,
      hit.agency,
    ) ??
    firstNonEmpty(detail.agencyDetails?.agencyCode, hit.agencyCode) ??
    'Unknown federal agency'
  );
}

/**
 * Combines a `search2` hit with its `fetchOpportunity` detail into an
 * insertable grant row. `sourceUrl` is the schema's upsert key, so it's
 * derived purely from the opportunity id and stable across re-crawls.
 *
 * `geographicScope` is deliberately left `null`: federal opportunities are
 * national by default and Grants.gov doesn't give us a reliable field to
 * derive a narrower scope from — better to leave it unset than guess.
 */
export function normalizeGrantsGovOpportunity(
  hit: GrantsGovSearchHit,
  detail: GrantsGovOpportunityDetail,
): NewGrantInput {
  const synopsis = detail.synopsis;

  return {
    sourceUrl: `https://www.grants.gov/search-results-detail/${hit.id}`,
    source: 'grants_gov',
    title: detail.opportunityTitle ?? hit.title,
    funder: resolveFunder(hit, detail),
    synopsis: stripHtml(synopsis?.synopsisDesc),
    eligibilityEntityTypes: normalizeApplicantTypes(
      synopsis?.applicantTypes,
    ),
    geographicScope: null,
    programAreas: null,
    awardFloor: parseDollarAmount(synopsis?.awardFloor),
    awardCeiling: parseDollarAmount(synopsis?.awardCeiling),
    expectedAwardsCount: parseCount(synopsis?.expectedNumberOfAwards),
    openDate: parseGrantsGovDate(hit.openDate ?? synopsis?.postingDate),
    closeDate: parseGrantsGovDate(hit.closeDate ?? synopsis?.responseDate),
    matchRequirement: false,
    applicationEffortEstimate: 'unknown',
    applicationFormSupported: false,
    status: 'open',
    lastVerifiedAt: new Date(),
  };
}