diff --git a/apps/outreach-review/app/routes/_index.tsx b/apps/outreach-review/app/routes/_index.tsx index 051628f..8019290 100644 --- a/apps/outreach-review/app/routes/_index.tsx +++ b/apps/outreach-review/app/routes/_index.tsx @@ -11,8 +11,10 @@ import type { Route } from './+types/_index.js'; export async function loader({ request }: Route.LoaderArgs) { if (request.method === 'HEAD') return; - const matches = await serverListPendingReviewMatches(db); - return { matches }; + const url = new URL(request.url); + const source = url.searchParams.get('source') ?? undefined; + const matches = await serverListPendingReviewMatches(db, { source }); + return { matches, source: source ?? null }; } export async function action({ request }: Route.ActionArgs) { @@ -36,10 +38,30 @@ export async function action({ request }: Route.ActionArgs) { export default function ReviewQueue({ loaderData }: Route.ComponentProps) { const matches = loaderData?.matches ?? []; + const activeSource = loaderData?.source ?? null; return (
-

Grant Match Review Queue

+

Grant Match Review Queue

+ {matches.length === 0 ? (

@@ -55,6 +77,9 @@ export default function ReviewQueue({ loaderData }: Route.ComponentProps) { Grant + + Source + Score @@ -78,6 +103,11 @@ export default function ReviewQueue({ loaderData }: Route.ComponentProps) { {match.grantTitle} + + + {match.source === 'irs_990pf' ? 'foundation' : match.source} + + {match.totalScore} {match.easyWin ? 'Yes' : 'No'} diff --git a/apps/outreach-worker/src/workflows/ingest-grants.ts b/apps/outreach-worker/src/workflows/ingest-grants.ts index 009f8d8..2f1f6aa 100644 --- a/apps/outreach-worker/src/workflows/ingest-grants.ts +++ b/apps/outreach-worker/src/workflows/ingest-grants.ts @@ -16,6 +16,7 @@ import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk'; import type { schema } from '@novelpad/outreach-core'; import { serverInsertGrants, + serverListGrantSourceUrls, type NewGrantInput, } from '@novelpad/outreach-core/server'; import type { NodePgDatabase } from 'drizzle-orm/node-postgres'; @@ -35,7 +36,7 @@ export type OutreachDb = NodePgDatabase; * endpoint per run. Search hits are cheap (one paginated request per ~100), * so this is generous headroom above the actual nightly posting volume. */ -const SEARCH_HIT_CAP = 1000; +const SEARCH_HIT_CAP = 2_000; /** * Cap on how many opportunities get a `fetchOpportunity` detail call per * run. Detail fetches are one request per grant (plus a politeness delay), @@ -92,14 +93,22 @@ interface HitWithDetail { * this run. */ async function fetchGrantDetails( + db: OutreachDb, hits: ReadonlyArray, ): Promise> { - const toFetch = hits.slice(0, DETAIL_FETCH_CAP); - if (hits.length > toFetch.length) { - console.warn( - `[ingest-grants] detail-fetch cap ${DETAIL_FETCH_CAP} reached: dropping ${hits.length - toFetch.length} of ${hits.length} opportunities this run`, - ); - } + // Spend the per-run detail budget on NEW opportunities first — the old + // hits.slice(0, cap) re-fetched the same head of the search results + // every night and never drained the backlog. Known opportunities fill + // any remaining budget (refreshing close dates / lastVerifiedAt). + const known = await serverListGrantSourceUrls(db, 'grants_gov'); + const isKnown = (hit: GrantsGovSearchHit) => + known.has(`https://www.grants.gov/search-results-detail/${hit.id}`); + const fresh = hits.filter((h) => !isKnown(h)); + const refresh = hits.filter(isKnown); + const toFetch = [...fresh, ...refresh].slice(0, DETAIL_FETCH_CAP); + console.log( + `[ingest-grants] detail budget ${DETAIL_FETCH_CAP}: ${Math.min(fresh.length, DETAIL_FETCH_CAP)} new, ${Math.max(0, Math.min(DETAIL_FETCH_CAP - fresh.length, refresh.length))} refresh, backlog remaining ${Math.max(0, fresh.length - DETAIL_FETCH_CAP)}`, + ); const details = await fetchOpportunityDetails(toFetch.map((hit) => hit.id)); return toFetch.map((hit, i) => { @@ -147,7 +156,7 @@ async function runIngestGrants(): Promise { const { db } = getIngestGrantsDeps(); const hits = await searchGrantsGovStep(); - const pairs = await fetchGrantDetailsStep(hits); + const pairs = await fetchGrantDetailsStep(db, hits); const normalized = normalize(pairs); await upsertGrantsStep(db, normalized); } diff --git a/packages/outreach-core/src/grants/queries/index.server.ts b/packages/outreach-core/src/grants/queries/index.server.ts index 9872dd7..b8b27f0 100644 --- a/packages/outreach-core/src/grants/queries/index.server.ts +++ b/packages/outreach-core/src/grants/queries/index.server.ts @@ -1,3 +1,4 @@ export * from './list-open-grants.server.js'; export * from './list-grants-needing-embedding.server.js'; export * from './list-eligible-grants-for-org.server.js'; +export * from './list-grant-source-urls.server.js'; diff --git a/packages/outreach-core/src/grants/queries/list-grant-source-urls.server.ts b/packages/outreach-core/src/grants/queries/list-grant-source-urls.server.ts new file mode 100644 index 0000000..75d7dde --- /dev/null +++ b/packages/outreach-core/src/grants/queries/list-grant-source-urls.server.ts @@ -0,0 +1,20 @@ +import { eq } from 'drizzle-orm'; + +import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; +import { schema } from '#~/db/db.js'; + +/** + * All source URLs already ingested for one source — lets ingestion spend + * its per-run detail-fetch budget on NEW opportunities first instead of + * re-fetching the same head of the search results every night. + */ +export async function serverListGrantSourceUrls( + db: NpOutreachDatabase | NpOutreachTransaction, + source: (typeof schema.grants.$inferSelect)['source'], +): Promise> { + const rows = await db + .select({ sourceUrl: schema.grants.sourceUrl }) + .from(schema.grants) + .where(eq(schema.grants.source, source)); + return new Set(rows.map((r) => r.sourceUrl)); +} diff --git a/packages/outreach-core/src/matches/queries/list-pending-review-matches.server.ts b/packages/outreach-core/src/matches/queries/list-pending-review-matches.server.ts index 80500f7..2acf84f 100644 --- a/packages/outreach-core/src/matches/queries/list-pending-review-matches.server.ts +++ b/packages/outreach-core/src/matches/queries/list-pending-review-matches.server.ts @@ -1,4 +1,4 @@ -import { desc, eq } from 'drizzle-orm'; +import { and, desc, eq } from 'drizzle-orm'; import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; import { schema } from '#~/db/db.js'; @@ -14,6 +14,7 @@ export interface PendingReviewMatch { orgName: string; grantTitle: string; funder: string; + source: string; totalScore: number; easyWin: boolean; isHero: boolean; @@ -22,6 +23,7 @@ export interface PendingReviewMatch { export async function serverListPendingReviewMatches( db: NpOutreachDatabase | NpOutreachTransaction, + { source }: { source?: string } = {}, ): Promise { return db .select({ @@ -29,6 +31,7 @@ export async function serverListPendingReviewMatches( orgName: schema.orgs.name, grantTitle: schema.grants.title, funder: schema.grants.funder, + source: schema.grants.source, totalScore: schema.matches.totalScore, easyWin: schema.matches.easyWin, isHero: schema.matches.isHero, @@ -37,7 +40,14 @@ export async function serverListPendingReviewMatches( .from(schema.matches) .innerJoin(schema.orgs, eq(schema.matches.orgId, schema.orgs.id)) .innerJoin(schema.grants, eq(schema.matches.grantId, schema.grants.id)) - .where(eq(schema.matches.reviewStatus, 'pending')) + .where( + source == null + ? eq(schema.matches.reviewStatus, 'pending') + : and( + eq(schema.matches.reviewStatus, 'pending'), + eq(schema.grants.source, source as never), + ), + ) // Reviewers see the best candidates first: heroes, then easy wins, // then raw score. Capped — nightly re-scoring generates thousands of // pending pairs and the queue is worked top-down, not exhaustively.