grant-outreach-engine/apps/outreach-worker/src/sources/pnd-rss/normalize.ts

/**
 * Pure normalization of a Philanthropy News Digest "RFPs" RSS feed body into
 * `NewGrantInput` rows.
 *
 * PND's RFP feed doesn't carry structured funder/deadline fields the way
 * Grants.gov's API does — everything upstream is prose inside `<title>` and
 * `<description>`. Funder and close-date extraction here are therefore
 * best-effort regex heuristics over that prose, not authoritative parses;
 * items that don't match fall back to a neutral placeholder (`'See RFP'`)
 * or `null` rather than guessing.
 */
import { XMLParser } from 'fast-xml-parser';

import type { NewGrantInput } from '@novelpad/outreach-core/server';

interface PndRssItem {
  readonly title?: string;
  readonly link?: string;
  readonly description?: string;
}

interface PndRssParsed {
  readonly rss?: {
    readonly channel?: {
      readonly item?: PndRssItem[];
    };
  };
}

// `isArray` forces `<item>` to always parse as an array, even when a feed
// has exactly one `<item>` — fast-xml-parser otherwise collapses a
// single-element repeated tag down to a bare object.
const parser = new XMLParser({
  ignoreAttributes: true,
  trimValues: true,
  isArray: (_name, jpath) => jpath === 'rss.channel.item',
});

/** Coerces a possibly-singular parsed `<item>` node into an array. Belt-and-suspenders alongside `isArray` above. */
function coerceArray(
  value: PndRssItem | PndRssItem[] | undefined,
): readonly PndRssItem[] {
  if (value == null) return [];
  return Array.isArray(value) ? value : [value];
}

const HTML_TAG_PATTERN = /<[^>]*>/g;
const NAMED_ENTITY_DECODERS: ReadonlyArray<readonly [RegExp, string]> = [
  [/&nbsp;/gi, ' '],
  [/&amp;/gi, '&'],
  [/&lt;/gi, '<'],
  [/&gt;/gi, '>'],
  [/&quot;/gi, '"'],
  [/&(?:#39|apos);/gi, "'"],
];

/** Strips HTML markup and decodes common entities from RSS `<description>` bodies. */
function stripHtml(html: string): string {
  let text = html.replace(HTML_TAG_PATTERN, ' ');
  for (const [pattern, replacement] of NAMED_ENTITY_DECODERS) {
    text = text.replace(pattern, replacement);
  }
  // Numeric entities, e.g. &#8217; (right single quote).
  text = text.replace(/&#(\d+);/g, (_match, code: string) =>
    String.fromCharCode(Number(code)),
  );
  return text.replace(/\s+/g, ' ').trim();
}

// Matches the leading phrase of a PND blurb up to the verb announcing the
// call for applications, e.g. "The Ford Foundation is accepting
// applications for..." -> "The Ford Foundation".
const FUNDER_PATTERN =
  /^(.+?)\s+(?:is\s+|are\s+|has\s+|have\s+)?(?:now\s+)?(?:accepting|invites?|seeks?|announces?|offers?)\b/i;

function matchFunder(text: string): string | null {
  const match = text.match(FUNDER_PATTERN);
  const candidate = match?.[1]?.trim();
  return candidate != null && candidate.length > 0 ? candidate : null;
}

/** Best-effort funder extraction: try the title first, then the (stripped) description, else fall back. */
function extractFunder(title: string, synopsis: string): string {
  return matchFunder(title) ?? matchFunder(synopsis) ?? 'See RFP';
}

// Best-effort deadline extraction over free-text prose. Ordered
// most-specific-first; the first pattern to match and parse to a valid
// Date wins.
const DEADLINE_PATTERNS: ReadonlyArray<RegExp> = [
  /deadline:?\s*(?:is\s*)?([A-Za-z]+\.?\s+\d{1,2},?\s+\d{4})/i,
  /deadline:?\s*(?:is\s*)?(\d{4}-\d{2}-\d{2})/i,
  /due\s+(?:date\s+(?:is\s+)?)?(?:on\s+)?([A-Za-z]+\.?\s+\d{1,2},?\s+\d{4})/i,
  /due\s+(?:date\s+(?:is\s+)?)?(?:on\s+)?(\d{4}-\d{2}-\d{2})/i,
];

function extractCloseDate(text: string): Date | null {
  for (const pattern of DEADLINE_PATTERNS) {
    const match = text.match(pattern);
    const dateText = match?.[1];
    if (dateText == null) continue;
    const parsed = new Date(dateText);
    if (!Number.isNaN(parsed.getTime())) return parsed;
  }
  return null;
}

/**
 * Parses a raw PND RFP RSS feed body into upsert-ready grant rows.
 *
 * Items with no `<link>` are skipped — `sourceUrl` is the upsert key in
 * `serverInsertGrants`, so a linkless item has nothing stable to key on.
 */
export function parsePndFeed(xml: string): NewGrantInput[] {
  const parsed = parser.parse(xml) as PndRssParsed;
  const items = coerceArray(parsed.rss?.channel?.item);

  const grants: NewGrantInput[] = [];
  for (const item of items) {
    const link = typeof item.link === 'string' ? item.link.trim() : '';
    if (link.length === 0) continue;

    const title = typeof item.title === 'string' ? item.title.trim() : '';
    const rawDescription =
      typeof item.description === 'string' ? item.description : '';
    const synopsis = stripHtml(rawDescription);

    grants.push({
      funder: extractFunder(title, synopsis),
      title: title.length > 0 ? title : 'Untitled RFP',
      sourceUrl: link,
      synopsis: synopsis.length > 0 ? synopsis : null,
      closeDate: extractCloseDate(`${title} ${synopsis}`),
      openDate: null,
      applicationEffortEstimate: 'unknown',
      source: 'pnd_rss',
      lastVerifiedAt: new Date(),
    });
  }
  return grants;
}