fix(ingestion): drain the Grants.gov backlog + source-filtered review queue

ingest-grants spent its 200-detail budget on hits.slice(0, 200) — the
same head of the search results every night; the backlog never drained.
Now new opportunities fill the budget first (serverListGrantSourceUrls
partition), remaining budget refreshes known ones; search cap raised to
2,000. Full eligible pool turns out to be 565 federal opportunities —
drained in two passes, 366 newly embedded.

Review queue gains a source badge column and All/Foundations/Federal
RFPs filter (?source=) — foundation easy-wins otherwise bury posted-RFP
matches, which score lower by design (no precedent, national pools) but
are the deadline-driven sends. Immediate proof: DOJ OVW FY2026 DV
program (closes 9/8) matched five NH domestic-violence orgs.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/outreach-review/app/routes/_index.tsx

@@ -11,8 +11,10 @@ import type { Route } from './+types/_index.js';
 export async function loader({ request }: Route.LoaderArgs) {
   if (request.method === 'HEAD') return;
 
-  const matches = await serverListPendingReviewMatches(db);
-  return { matches };
+  const url = new URL(request.url);
+  const source = url.searchParams.get('source') ?? undefined;
+  const matches = await serverListPendingReviewMatches(db, { source });
+  return { matches, source: source ?? null };
 }
 
 export async function action({ request }: Route.ActionArgs) {
@@ -36,10 +38,30 @@ export async function action({ request }: Route.ActionArgs) {
 
 export default function ReviewQueue({ loaderData }: Route.ComponentProps) {
   const matches = loaderData?.matches ?? [];
+  const activeSource = loaderData?.source ?? null;
 
   return (
     <main className="container mx-auto p-6">
-      <h1 className="mb-4 text-2xl font-semibold">Grant Match Review Queue</h1>
+      <h1 className="mb-2 text-2xl font-semibold">Grant Match Review Queue</h1>
+      <nav className="mb-4 flex gap-3 text-sm" aria-label="Filter by source">
+        {[
+          [null, 'All'],
+          ['irs_990pf', 'Foundations'],
+          ['grants_gov', 'Federal RFPs'],
+        ].map(([value, label]) => (
+          <a
+            key={label as string}
+            href={value == null ? '/' : `/?source=${value}`}
+            className={
+              activeSource === value
+                ? 'font-semibold underline'
+                : 'text-blue-700 underline'
+            }
+          >
+            {label}
+          </a>
+        ))}
+      </nav>
 
       {matches.length === 0 ? (
         <p role="status" className="text-gray-600">
@@ -55,6 +77,9 @@ export default function ReviewQueue({ loaderData }: Route.ComponentProps) {
               <th scope="col" className="p-2">
                 Grant
               </th>
+              <th scope="col" className="p-2">
+                Source
+              </th>
               <th scope="col" className="p-2">
                 Score
               </th>
@@ -78,6 +103,11 @@ export default function ReviewQueue({ loaderData }: Route.ComponentProps) {
                   </Link>
                 </td>
                 <td className="p-2">{match.grantTitle}</td>
+                <td className="p-2">
+                  <span className="rounded bg-gray-200 px-2 py-0.5 text-xs">
+                    {match.source === 'irs_990pf' ? 'foundation' : match.source}
+                  </span>
+                </td>
                 <td className="p-2">{match.totalScore}</td>
                 <td className="p-2">{match.easyWin ? 'Yes' : 'No'}</td>
                 <td className="p-2">

apps/outreach-worker/src/workflows/ingest-grants.ts

@@ -16,6 +16,7 @@ import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
 import type { schema } from '@novelpad/outreach-core';
 import {
   serverInsertGrants,
+  serverListGrantSourceUrls,
   type NewGrantInput,
 } from '@novelpad/outreach-core/server';
 import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -35,7 +36,7 @@ export type OutreachDb = NodePgDatabase<typeof schema>;
  * endpoint per run. Search hits are cheap (one paginated request per ~100),
  * so this is generous headroom above the actual nightly posting volume.
  */
-const SEARCH_HIT_CAP = 1000;
+const SEARCH_HIT_CAP = 2_000;
 /**
  * Cap on how many opportunities get a `fetchOpportunity` detail call per
  * run. Detail fetches are one request per grant (plus a politeness delay),
@@ -92,14 +93,22 @@ interface HitWithDetail {
  * this run.
  */
 async function fetchGrantDetails(
+  db: OutreachDb,
   hits: ReadonlyArray<GrantsGovSearchHit>,
 ): Promise<ReadonlyArray<HitWithDetail>> {
-  const toFetch = hits.slice(0, DETAIL_FETCH_CAP);
-  if (hits.length > toFetch.length) {
-    console.warn(
-      `[ingest-grants] detail-fetch cap ${DETAIL_FETCH_CAP} reached: dropping ${hits.length - toFetch.length} of ${hits.length} opportunities this run`,
-    );
-  }
+  // Spend the per-run detail budget on NEW opportunities first — the old
+  // hits.slice(0, cap) re-fetched the same head of the search results
+  // every night and never drained the backlog. Known opportunities fill
+  // any remaining budget (refreshing close dates / lastVerifiedAt).
+  const known = await serverListGrantSourceUrls(db, 'grants_gov');
+  const isKnown = (hit: GrantsGovSearchHit) =>
+    known.has(`https://www.grants.gov/search-results-detail/${hit.id}`);
+  const fresh = hits.filter((h) => !isKnown(h));
+  const refresh = hits.filter(isKnown);
+  const toFetch = [...fresh, ...refresh].slice(0, DETAIL_FETCH_CAP);
+  console.log(
+    `[ingest-grants] detail budget ${DETAIL_FETCH_CAP}: ${Math.min(fresh.length, DETAIL_FETCH_CAP)} new, ${Math.max(0, Math.min(DETAIL_FETCH_CAP - fresh.length, refresh.length))} refresh, backlog remaining ${Math.max(0, fresh.length - DETAIL_FETCH_CAP)}`,
+  );
 
   const details = await fetchOpportunityDetails(toFetch.map((hit) => hit.id));
   return toFetch.map((hit, i) => {
@@ -147,7 +156,7 @@ async function runIngestGrants(): Promise<void> {
   const { db } = getIngestGrantsDeps();
 
   const hits = await searchGrantsGovStep();
-  const pairs = await fetchGrantDetailsStep(hits);
+  const pairs = await fetchGrantDetailsStep(db, hits);
   const normalized = normalize(pairs);
   await upsertGrantsStep(db, normalized);
 }