fix(ingestion): drain the Grants.gov backlog + source-filtered review queue

ingest-grants spent its 200-detail budget on hits.slice(0, 200) — the
same head of the search results every night; the backlog never drained.
Now new opportunities fill the budget first (serverListGrantSourceUrls
partition), remaining budget refreshes known ones; search cap raised to
2,000. Full eligible pool turns out to be 565 federal opportunities —
drained in two passes, 366 newly embedded.

Review queue gains a source badge column and All/Foundations/Federal
RFPs filter (?source=) — foundation easy-wins otherwise bury posted-RFP
matches, which score lower by design (no precedent, national pools) but
are the deadline-driven sends. Immediate proof: DOJ OVW FY2026 DV
program (closes 9/8) matched five NH domestic-violence orgs.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/outreach-review/app/routes/_index.tsx

@@ -11,8 +11,10 @@ import type { Route } from './+types/_index.js';
 export async function loader({ request }: Route.LoaderArgs) {
   if (request.method === 'HEAD') return;
 
-  const matches = await serverListPendingReviewMatches(db);
+  const url = new URL(request.url);
-  return { matches };
+  const source = url.searchParams.get('source') ?? undefined;
+  const matches = await serverListPendingReviewMatches(db, { source });
+  return { matches, source: source ?? null };
 }
 
 export async function action({ request }: Route.ActionArgs) {
@@ -36,10 +38,30 @@ export async function action({ request }: Route.ActionArgs) {
 
 export default function ReviewQueue({ loaderData }: Route.ComponentProps) {
   const matches = loaderData?.matches ?? [];
+  const activeSource = loaderData?.source ?? null;
 
   return (
     <main className="container mx-auto p-6">
-      <h1 className="mb-4 text-2xl font-semibold">Grant Match Review Queue</h1>
+      <h1 className="mb-2 text-2xl font-semibold">Grant Match Review Queue</h1>
+      <nav className="mb-4 flex gap-3 text-sm" aria-label="Filter by source">
+        {[
+          [null, 'All'],
+          ['irs_990pf', 'Foundations'],
+          ['grants_gov', 'Federal RFPs'],
+        ].map(([value, label]) => (
+          <a
+            key={label as string}
+            href={value == null ? '/' : `/?source=${value}`}
+            className={
+              activeSource === value
+                ? 'font-semibold underline'
+                : 'text-blue-700 underline'
+            }
+          >
+            {label}
+          </a>
+        ))}
+      </nav>
 
       {matches.length === 0 ? (
         <p role="status" className="text-gray-600">
@@ -55,6 +77,9 @@ export default function ReviewQueue({ loaderData }: Route.ComponentProps) {
               <th scope="col" className="p-2">
                 Grant
               </th>
+              <th scope="col" className="p-2">
+                Source
+              </th>
               <th scope="col" className="p-2">
                 Score
               </th>
@@ -78,6 +103,11 @@ export default function ReviewQueue({ loaderData }: Route.ComponentProps) {
                   </Link>
                 </td>
                 <td className="p-2">{match.grantTitle}</td>
+                <td className="p-2">
+                  <span className="rounded bg-gray-200 px-2 py-0.5 text-xs">
+                    {match.source === 'irs_990pf' ? 'foundation' : match.source}
+                  </span>
+                </td>
                 <td className="p-2">{match.totalScore}</td>
                 <td className="p-2">{match.easyWin ? 'Yes' : 'No'}</td>
                 <td className="p-2">

apps/outreach-worker/src/workflows/ingest-grants.ts

@@ -16,6 +16,7 @@ import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
 import type { schema } from '@novelpad/outreach-core';
 import {
   serverInsertGrants,
+  serverListGrantSourceUrls,
   type NewGrantInput,
 } from '@novelpad/outreach-core/server';
 import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -35,7 +36,7 @@ export type OutreachDb = NodePgDatabase<typeof schema>;
  * endpoint per run. Search hits are cheap (one paginated request per ~100),
  * so this is generous headroom above the actual nightly posting volume.
  */
-const SEARCH_HIT_CAP = 1000;
+const SEARCH_HIT_CAP = 2_000;
 /**
  * Cap on how many opportunities get a `fetchOpportunity` detail call per
  * run. Detail fetches are one request per grant (plus a politeness delay),
@@ -92,14 +93,22 @@ interface HitWithDetail {
  * this run.
  */
 async function fetchGrantDetails(
+  db: OutreachDb,
   hits: ReadonlyArray<GrantsGovSearchHit>,
 ): Promise<ReadonlyArray<HitWithDetail>> {
-  const toFetch = hits.slice(0, DETAIL_FETCH_CAP);
+  // Spend the per-run detail budget on NEW opportunities first — the old
-  if (hits.length > toFetch.length) {
+  // hits.slice(0, cap) re-fetched the same head of the search results
-    console.warn(
+  // every night and never drained the backlog. Known opportunities fill
-      `[ingest-grants] detail-fetch cap ${DETAIL_FETCH_CAP} reached: dropping ${hits.length - toFetch.length} of ${hits.length} opportunities this run`,
+  // any remaining budget (refreshing close dates / lastVerifiedAt).
-    );
+  const known = await serverListGrantSourceUrls(db, 'grants_gov');
-  }
+  const isKnown = (hit: GrantsGovSearchHit) =>
+    known.has(`https://www.grants.gov/search-results-detail/${hit.id}`);
+  const fresh = hits.filter((h) => !isKnown(h));
+  const refresh = hits.filter(isKnown);
+  const toFetch = [...fresh, ...refresh].slice(0, DETAIL_FETCH_CAP);
+  console.log(
+    `[ingest-grants] detail budget ${DETAIL_FETCH_CAP}: ${Math.min(fresh.length, DETAIL_FETCH_CAP)} new, ${Math.max(0, Math.min(DETAIL_FETCH_CAP - fresh.length, refresh.length))} refresh, backlog remaining ${Math.max(0, fresh.length - DETAIL_FETCH_CAP)}`,
+  );
 
   const details = await fetchOpportunityDetails(toFetch.map((hit) => hit.id));
   return toFetch.map((hit, i) => {
@@ -147,7 +156,7 @@ async function runIngestGrants(): Promise<void> {
   const { db } = getIngestGrantsDeps();
 
   const hits = await searchGrantsGovStep();
-  const pairs = await fetchGrantDetailsStep(hits);
+  const pairs = await fetchGrantDetailsStep(db, hits);
   const normalized = normalize(pairs);
   await upsertGrantsStep(db, normalized);
 }

packages/outreach-core/src/grants/queries/index.server.ts

@@ -1,3 +1,4 @@
 export * from './list-open-grants.server.js';
 export * from './list-grants-needing-embedding.server.js';
 export * from './list-eligible-grants-for-org.server.js';
+export * from './list-grant-source-urls.server.js';

packages/outreach-core/src/grants/queries/list-grant-source-urls.server.ts

@@ -0,0 +1,20 @@
+import { eq } from 'drizzle-orm';
+
+import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
+import { schema } from '#~/db/db.js';
+
+/**
+ * All source URLs already ingested for one source — lets ingestion spend
+ * its per-run detail-fetch budget on NEW opportunities first instead of
+ * re-fetching the same head of the search results every night.
+ */
+export async function serverListGrantSourceUrls(
+  db: NpOutreachDatabase | NpOutreachTransaction,
+  source: (typeof schema.grants.$inferSelect)['source'],
+): Promise<Set<string>> {
+  const rows = await db
+    .select({ sourceUrl: schema.grants.sourceUrl })
+    .from(schema.grants)
+    .where(eq(schema.grants.source, source));
+  return new Set(rows.map((r) => r.sourceUrl));
+}

packages/outreach-core/src/matches/queries/list-pending-review-matches.server.ts

@@ -1,4 +1,4 @@
-import { desc, eq } from 'drizzle-orm';
+import { and, desc, eq } from 'drizzle-orm';
 
 import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
 import { schema } from '#~/db/db.js';
@@ -14,6 +14,7 @@ export interface PendingReviewMatch {
   orgName: string;
   grantTitle: string;
   funder: string;
+  source: string;
   totalScore: number;
   easyWin: boolean;
   isHero: boolean;
@@ -22,6 +23,7 @@ export interface PendingReviewMatch {
 
 export async function serverListPendingReviewMatches(
   db: NpOutreachDatabase | NpOutreachTransaction,
+  { source }: { source?: string } = {},
 ): Promise<PendingReviewMatch[]> {
   return db
     .select({
@@ -29,6 +31,7 @@ export async function serverListPendingReviewMatches(
       orgName: schema.orgs.name,
       grantTitle: schema.grants.title,
       funder: schema.grants.funder,
+      source: schema.grants.source,
       totalScore: schema.matches.totalScore,
       easyWin: schema.matches.easyWin,
       isHero: schema.matches.isHero,
@@ -37,7 +40,14 @@ export async function serverListPendingReviewMatches(
     .from(schema.matches)
     .innerJoin(schema.orgs, eq(schema.matches.orgId, schema.orgs.id))
     .innerJoin(schema.grants, eq(schema.matches.grantId, schema.grants.id))
-    .where(eq(schema.matches.reviewStatus, 'pending'))
+    .where(
+      source == null
+        ? eq(schema.matches.reviewStatus, 'pending')
+        : and(
+            eq(schema.matches.reviewStatus, 'pending'),
+            eq(schema.grants.source, source as never),
+          ),
+    )
     // Reviewers see the best candidates first: heroes, then easy wins,
     // then raw score. Capped — nightly re-scoring generates thousands of
     // pending pairs and the queue is worked top-down, not exhaustively.