feat(scoring): federal funder precedent via USASpending (ALN-level)
Precedent was always a valid criterion for federal grants — the index just didn't cover them. Now: ingest captures ALN/CFDA numbers from fetchOpportunity (grants.alns); nightly federalPrecedent workflow (04:45) queries USASpending award search per distinct program (free official API, 3-year NH lookback) and stamps program_state_award_count + sample recipients onto open grants (multi-ALN keeps highest). Match retrieval feeds the same funderStateGrantCount input and 25-point tiers foundations use; detail page shows the recipients-evidence table with an incumbent-renewal caution. Also: detail refresh now rotates oldest-verified-first (serverMapGrantVerification) — the Set-based partition re-fetched the same 200 every pass, leaving 365/565 grants ALN-less. Live: 564/564 grants ALN-tagged, 156 programs swept, 85 with NH history, 468 grants carrying precedent, first federal easy-wins (66pts, 25/25 precedent, Aug-24 deadline). 158 tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -290,11 +290,15 @@ export default function MatchDetail({ loaderData }: Route.ComponentProps) {
|
||||
{nhHistory.length > 0 && (
|
||||
<section className="rounded border p-4">
|
||||
<h2 className="mb-1 text-lg font-semibold">
|
||||
Funder giving history (from 990-PF filings)
|
||||
{grant.source === 'irs_990pf'
|
||||
? 'Funder giving history (from 990-PF filings)'
|
||||
: 'Recent program awards to NH recipients (USASpending)'}
|
||||
</h2>
|
||||
<p className="mb-3 text-sm text-gray-600">
|
||||
The concrete evidence behind the precedent score — who this
|
||||
funder actually paid, sorted in-state first.
|
||||
The concrete evidence behind the precedent score —{' '}
|
||||
{grant.source === 'irs_990pf'
|
||||
? 'who this funder actually paid, sorted in-state first.'
|
||||
: 'who this federal program actually funded in NH recently. Watch for one incumbent recapturing renewals vs. genuine spread across orgs.'}
|
||||
</p>
|
||||
<table className="w-full text-sm">
|
||||
<thead>
|
||||
|
||||
@@ -33,6 +33,7 @@ import pg from 'pg';
|
||||
import { setEmbedGrantsDeps } from './workflows/embed-grants.js';
|
||||
import { setEnrichOrgsDeps } from './workflows/enrich-orgs.js';
|
||||
import { setExpireGrantsDeps } from './workflows/expire-grants.js';
|
||||
import { setFederalPrecedentDeps } from './workflows/federal-precedent.js';
|
||||
import { setIngestGrantsDeps } from './workflows/ingest-grants.js';
|
||||
import { setIngest990pfDeps } from './workflows/ingest-990pf.js';
|
||||
import { setIngestNhdojOrgsDeps } from './workflows/ingest-nhdoj-orgs.js';
|
||||
@@ -71,6 +72,7 @@ async function main() {
|
||||
setEmbedGrantsDeps({ db });
|
||||
setMatchGrantsDeps({ db });
|
||||
setIngest990pfDeps({ db });
|
||||
setFederalPrecedentDeps({ db });
|
||||
|
||||
DBOS.setConfig({
|
||||
name: 'helmdocs-outreach-worker',
|
||||
|
||||
@@ -28,6 +28,10 @@ import {
|
||||
runExpireGrantsNow,
|
||||
setExpireGrantsDeps,
|
||||
} from './workflows/expire-grants.js';
|
||||
import {
|
||||
runFederalPrecedentNow,
|
||||
setFederalPrecedentDeps,
|
||||
} from './workflows/federal-precedent.js';
|
||||
import {
|
||||
runIngestGrantsNow,
|
||||
setIngestGrantsDeps,
|
||||
@@ -58,6 +62,7 @@ const RUNNERS: Record<string, () => Promise<void>> = {
|
||||
embedGrants: runEmbedGrantsNow,
|
||||
matchGrants: runMatchGrantsNow,
|
||||
ingest990pf: runIngest990PfNow,
|
||||
federalPrecedent: runFederalPrecedentNow,
|
||||
};
|
||||
|
||||
const FIRST_RUN_ORDER = [
|
||||
@@ -67,6 +72,7 @@ const FIRST_RUN_ORDER = [
|
||||
'expireGrants',
|
||||
'enrichOrgs',
|
||||
'ingest990pf',
|
||||
'federalPrecedent',
|
||||
'embedGrants',
|
||||
'matchGrants',
|
||||
];
|
||||
@@ -106,6 +112,7 @@ async function main() {
|
||||
setEmbedGrantsDeps({ db });
|
||||
setMatchGrantsDeps({ db });
|
||||
setIngest990pfDeps({ db });
|
||||
setFederalPrecedentDeps({ db });
|
||||
|
||||
DBOS.setConfig({
|
||||
name: 'helmdocs-outreach-worker',
|
||||
|
||||
@@ -82,6 +82,11 @@ export interface GrantsGovOpportunityDetail {
|
||||
readonly agencyCode?: string | null;
|
||||
} | null;
|
||||
readonly synopsis: GrantsGovSynopsis | null;
|
||||
/** Assistance Listing (CFDA) entries, e.g. [{ cfdaNumber: '93.243' }]. */
|
||||
readonly cfdas?: ReadonlyArray<{
|
||||
readonly cfdaNumber?: string | null;
|
||||
readonly programTitle?: string | null;
|
||||
}> | null;
|
||||
}
|
||||
|
||||
interface GrantsGovDetailResponseEnvelope {
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
searchHitFixture,
|
||||
} from '#~/sources/grants-gov/fixtures.js';
|
||||
import {
|
||||
normalizeAlns,
|
||||
normalizeApplicantTypes,
|
||||
normalizeGrantsGovOpportunity,
|
||||
parseCount,
|
||||
@@ -232,3 +233,23 @@ describe('normalizeApplicantTypes', () => {
|
||||
expect(normalizeApplicantTypes([])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeAlns', () => {
|
||||
it('extracts distinct valid ALN numbers', () => {
|
||||
expect(
|
||||
normalizeAlns([
|
||||
{ cfdaNumber: '93.243' },
|
||||
{ cfdaNumber: ' 93.243 ' },
|
||||
{ cfdaNumber: '16.888' },
|
||||
{ cfdaNumber: 'garbage' },
|
||||
{ cfdaNumber: null },
|
||||
]),
|
||||
).toEqual(['93.243', '16.888']);
|
||||
});
|
||||
|
||||
it('returns null for empty or missing lists', () => {
|
||||
expect(normalizeAlns(null)).toBeNull();
|
||||
expect(normalizeAlns([])).toBeNull();
|
||||
expect(normalizeAlns([{ cfdaNumber: 'x' }])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -98,6 +98,24 @@ export function stripHtml(html: string | null | undefined): string | null {
|
||||
}
|
||||
|
||||
/** Extracts non-empty eligibility descriptions from `synopsis.applicantTypes`. */
|
||||
/** Distinct, trimmed ALN/CFDA numbers ('93.243') from the detail's cfdas list. */
|
||||
export function normalizeAlns(
|
||||
cfdas:
|
||||
| ReadonlyArray<{ readonly cfdaNumber?: string | null }>
|
||||
| null
|
||||
| undefined,
|
||||
): string[] | null {
|
||||
if (cfdas == null) return null;
|
||||
const nums = [
|
||||
...new Set(
|
||||
cfdas
|
||||
.map((c) => c.cfdaNumber?.trim())
|
||||
.filter((v): v is string => v != null && /^\d{2}\.\d{3}$/.test(v)),
|
||||
),
|
||||
];
|
||||
return nums.length > 0 ? nums : null;
|
||||
}
|
||||
|
||||
export function normalizeApplicantTypes(
|
||||
applicantTypes: ReadonlyArray<GrantsGovApplicantType> | null | undefined,
|
||||
): string[] | null {
|
||||
@@ -162,6 +180,7 @@ export function normalizeGrantsGovOpportunity(
|
||||
title: detail.opportunityTitle ?? hit.title,
|
||||
funder: resolveFunder(hit, detail),
|
||||
synopsis: stripHtml(synopsis?.synopsisDesc),
|
||||
alns: normalizeAlns(detail.cfdas),
|
||||
eligibilityEntityTypes: normalizeApplicantTypes(
|
||||
synopsis?.applicantTypes,
|
||||
),
|
||||
|
||||
93
apps/outreach-worker/src/sources/usaspending/client.ts
Normal file
93
apps/outreach-worker/src/sources/usaspending/client.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* USASpending.gov award-search client — the federal analog of the 990-PF
|
||||
* grants-paid index. Official API, free, no key.
|
||||
* Docs: https://api.usaspending.gov/docs/endpoints
|
||||
*/
|
||||
|
||||
const SEARCH_URL =
|
||||
'https://api.usaspending.gov/api/v2/search/spending_by_award/';
|
||||
|
||||
/** Grant-shaped assistance award type codes (block/formula/project/coop). */
|
||||
const GRANT_AWARD_TYPE_CODES = ['02', '03', '04', '05'];
|
||||
|
||||
const DEFAULT_POLITENESS_DELAY_MS = 300;
|
||||
|
||||
export interface ProgramStateAwardSample {
|
||||
readonly recipientName: string;
|
||||
readonly amount: number | null;
|
||||
readonly startDate: string | null;
|
||||
}
|
||||
|
||||
export interface ProgramStateAwards {
|
||||
readonly aln: string;
|
||||
readonly awardCount: number;
|
||||
readonly samples: ProgramStateAwardSample[];
|
||||
}
|
||||
|
||||
export interface UsaSpendingClientOptions {
|
||||
readonly fetchImpl?: typeof fetch;
|
||||
readonly politenessDelayMs?: number;
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Recent grant awards under one Assistance Listing to recipients in one
|
||||
* state: total count plus a handful of sample recipients (review-page
|
||||
* evidence — lets a human spot "one incumbent's renewals" vs "spread
|
||||
* across orgs like ours").
|
||||
*/
|
||||
export async function fetchProgramStateAwards(
|
||||
aln: string,
|
||||
state: string,
|
||||
{ startDate, endDate }: { startDate: string; endDate: string },
|
||||
options: UsaSpendingClientOptions = {},
|
||||
): Promise<ProgramStateAwards> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch;
|
||||
|
||||
const res = await fetchImpl(SEARCH_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
filters: {
|
||||
award_type_codes: GRANT_AWARD_TYPE_CODES,
|
||||
recipient_locations: [{ country: 'USA', state }],
|
||||
time_period: [{ start_date: startDate, end_date: endDate }],
|
||||
program_numbers: [aln],
|
||||
},
|
||||
fields: ['Award ID', 'Recipient Name', 'Award Amount', 'Start Date'],
|
||||
limit: 10,
|
||||
page: 1,
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`USASpending search failed for ALN ${aln} (${state}): ${res.status} ${res.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const body = (await res.json()) as {
|
||||
results?: Array<{
|
||||
'Recipient Name'?: string | null;
|
||||
'Award Amount'?: number | null;
|
||||
'Start Date'?: string | null;
|
||||
}>;
|
||||
page_metadata?: { total?: number | null };
|
||||
};
|
||||
|
||||
await wait(options.politenessDelayMs ?? DEFAULT_POLITENESS_DELAY_MS);
|
||||
|
||||
const samples: ProgramStateAwardSample[] = (body.results ?? []).map((r) => ({
|
||||
recipientName: r['Recipient Name'] ?? 'Unknown recipient',
|
||||
amount: r['Award Amount'] == null ? null : Math.round(r['Award Amount']),
|
||||
startDate: r['Start Date'] ?? null,
|
||||
}));
|
||||
|
||||
return {
|
||||
aln,
|
||||
awardCount: body.page_metadata?.total ?? samples.length,
|
||||
samples,
|
||||
};
|
||||
}
|
||||
194
apps/outreach-worker/src/workflows/federal-precedent.ts
Normal file
194
apps/outreach-worker/src/workflows/federal-precedent.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* Nightly federal-precedent workflow — USASpending award history per
|
||||
* Assistance Listing (ALN/CFDA), the federal analog of the 990-PF index.
|
||||
*
|
||||
* For each distinct ALN across open federal grants: count grant awards to
|
||||
* recipients in the target state over the last ~3 fiscal years, stamp the
|
||||
* count + sample recipients onto every open grant carrying that ALN.
|
||||
* Match retrieval feeds the count into the same 25-point precedent
|
||||
* subscore foundations use — "this program funded N NH orgs recently" and
|
||||
* "this foundation funded N NH orgs recently" are the same signal.
|
||||
*
|
||||
* Runs at 04:45 — after ingest (03:00, which refreshes ALNs) and before
|
||||
* matching (05:15). Registration follows `ingest-grants.ts` exactly.
|
||||
*/
|
||||
import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
|
||||
import type { schema } from '@novelpad/outreach-core';
|
||||
import {
|
||||
serverListOpenFederalAlns,
|
||||
serverResetFederalPrecedent,
|
||||
serverSetFederalPrecedent,
|
||||
} from '@novelpad/outreach-core/server';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import {
|
||||
fetchProgramStateAwards,
|
||||
type ProgramStateAwards,
|
||||
} from '#~/sources/usaspending/client.js';
|
||||
|
||||
export type OutreachDb = NodePgDatabase<typeof schema>;
|
||||
|
||||
const RECIPIENT_STATE = 'NH';
|
||||
/** ~3 fiscal years of history — recent enough to predict, long enough to see a pattern. */
|
||||
const LOOKBACK_YEARS = 3;
|
||||
|
||||
export interface FederalPrecedentDeps {
|
||||
readonly db: OutreachDb;
|
||||
}
|
||||
|
||||
let registeredDeps: FederalPrecedentDeps | null = null;
|
||||
|
||||
export function setFederalPrecedentDeps(deps: FederalPrecedentDeps): void {
|
||||
registeredDeps = deps;
|
||||
}
|
||||
|
||||
function getFederalPrecedentDeps(): FederalPrecedentDeps {
|
||||
if (registeredDeps == null) {
|
||||
throw new Error(
|
||||
'FederalPrecedentDeps not registered. Call setFederalPrecedentDeps() before DBOS.launch().',
|
||||
);
|
||||
}
|
||||
return registeredDeps;
|
||||
}
|
||||
|
||||
async function listAlns(db: OutreachDb): Promise<string[]> {
|
||||
return serverListOpenFederalAlns(db);
|
||||
}
|
||||
const listAlnsStep = DBOS.registerStep(listAlns, {
|
||||
name: 'listOpenFederalAlns',
|
||||
retriesAllowed: true,
|
||||
maxAttempts: 3,
|
||||
});
|
||||
|
||||
async function fetchAlnAwards(
|
||||
aln: string,
|
||||
startDate: string,
|
||||
endDate: string,
|
||||
): Promise<ProgramStateAwards> {
|
||||
return fetchProgramStateAwards(aln, RECIPIENT_STATE, {
|
||||
startDate,
|
||||
endDate,
|
||||
});
|
||||
}
|
||||
const fetchAlnAwardsStep = DBOS.registerStep(fetchAlnAwards, {
|
||||
name: 'fetchAlnStateAwards',
|
||||
retriesAllowed: true,
|
||||
maxAttempts: 3,
|
||||
});
|
||||
|
||||
async function applyPrecedent(
|
||||
db: OutreachDb,
|
||||
precedent: ProgramStateAwards,
|
||||
): Promise<void> {
|
||||
await serverSetFederalPrecedent(db, {
|
||||
aln: precedent.aln,
|
||||
awardCount: precedent.awardCount,
|
||||
samples: precedent.samples,
|
||||
});
|
||||
}
|
||||
const applyPrecedentStep = DBOS.registerStep(applyPrecedent, {
|
||||
name: 'applyFederalPrecedent',
|
||||
retriesAllowed: true,
|
||||
maxAttempts: 3,
|
||||
});
|
||||
|
||||
async function resetPrecedent(db: OutreachDb): Promise<void> {
|
||||
await serverResetFederalPrecedent(db);
|
||||
}
|
||||
const resetPrecedentStep = DBOS.registerStep(resetPrecedent, {
|
||||
name: 'resetFederalPrecedent',
|
||||
retriesAllowed: true,
|
||||
maxAttempts: 3,
|
||||
});
|
||||
|
||||
async function runFederalPrecedent(): Promise<void> {
|
||||
const { db } = getFederalPrecedentDeps();
|
||||
|
||||
const alns = await listAlnsStep(db);
|
||||
if (alns.length === 0) {
|
||||
console.log(
|
||||
'[federal-precedent] no open federal grants carry ALNs yet (ingest refresh pending?)',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const end = new Date();
|
||||
const start = new Date(end);
|
||||
start.setFullYear(start.getFullYear() - LOOKBACK_YEARS);
|
||||
const startDate = start.toISOString().slice(0, 10);
|
||||
const endDate = end.toISOString().slice(0, 10);
|
||||
|
||||
await resetPrecedentStep(db);
|
||||
|
||||
let programsWithHistory = 0;
|
||||
let failed = 0;
|
||||
for (const aln of alns) {
|
||||
try {
|
||||
const precedent = await fetchAlnAwardsStep(aln, startDate, endDate);
|
||||
if (precedent.awardCount > 0) {
|
||||
await applyPrecedentStep(db, precedent);
|
||||
programsWithHistory++;
|
||||
}
|
||||
} catch (err) {
|
||||
failed++;
|
||||
console.error(`[federal-precedent] ALN ${aln} failed:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[federal-precedent] programs=${alns.length} withNhHistory=${programsWithHistory} failed=${failed}`,
|
||||
);
|
||||
if (failed > 0 && failed / alns.length > 0.2) {
|
||||
throw new Error(
|
||||
`[federal-precedent] systemic failure: ${failed}/${alns.length} programs failed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachFederalPrecedentRegistered?: boolean;
|
||||
__outreachFederalPrecedentHandle?: (
|
||||
scheduledTime: Date,
|
||||
startedAt: Date,
|
||||
) => Promise<void>;
|
||||
};
|
||||
|
||||
if (!g.__outreachFederalPrecedentRegistered) {
|
||||
g.__outreachFederalPrecedentRegistered = true;
|
||||
|
||||
const federalPrecedent = async (_scheduledTime: Date, _startedAt: Date) => {
|
||||
try {
|
||||
await runFederalPrecedent();
|
||||
} catch (err) {
|
||||
console.error('[federal-precedent] pass failed:', err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// Must be registered as BOTH a workflow and a scheduled function,
|
||||
// referencing the same function object — see ingest-grants.ts.
|
||||
g.__outreachFederalPrecedentHandle = DBOS.registerWorkflow(federalPrecedent, {
|
||||
name: 'federalPrecedent',
|
||||
});
|
||||
DBOS.registerScheduled(federalPrecedent, {
|
||||
crontab: '45 4 * * *',
|
||||
name: 'federalPrecedent',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts one durable run of this workflow immediately through DBOS —
|
||||
* the exact production path (workflow + checkpointed steps), used by
|
||||
* `run-once.ts` for supervised/manual passes. Requires deps injected and
|
||||
* `DBOS.launch()` completed.
|
||||
*/
|
||||
export function runFederalPrecedentNow(): Promise<void> {
|
||||
const handle = g.__outreachFederalPrecedentHandle;
|
||||
if (handle == null) {
|
||||
throw new Error(
|
||||
'federalPrecedent is not registered; was this module imported before DBOS.launch()?',
|
||||
);
|
||||
}
|
||||
return handle(new Date(), new Date());
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
|
||||
import type { schema } from '@novelpad/outreach-core';
|
||||
import {
|
||||
serverInsertGrants,
|
||||
serverListGrantSourceUrls,
|
||||
serverMapGrantVerification,
|
||||
type NewGrantInput,
|
||||
} from '@novelpad/outreach-core/server';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
@@ -100,11 +100,19 @@ async function fetchGrantDetails(
|
||||
// hits.slice(0, cap) re-fetched the same head of the search results
|
||||
// every night and never drained the backlog. Known opportunities fill
|
||||
// any remaining budget (refreshing close dates / lastVerifiedAt).
|
||||
const known = await serverListGrantSourceUrls(db, 'grants_gov');
|
||||
const isKnown = (hit: GrantsGovSearchHit) =>
|
||||
known.has(`https://www.grants.gov/search-results-detail/${hit.id}`);
|
||||
const fresh = hits.filter((h) => !isKnown(h));
|
||||
const refresh = hits.filter(isKnown);
|
||||
const known = await serverMapGrantVerification(db, 'grants_gov');
|
||||
const urlOf = (hit: GrantsGovSearchHit) =>
|
||||
`https://www.grants.gov/search-results-detail/${hit.id}`;
|
||||
const fresh = hits.filter((h) => !known.has(urlOf(h)));
|
||||
// Oldest-verified first so refreshes rotate through the whole corpus
|
||||
// instead of re-fetching the same head of the search results.
|
||||
const refresh = hits
|
||||
.filter((h) => known.has(urlOf(h)))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
(known.get(urlOf(a))?.getTime() ?? 0) -
|
||||
(known.get(urlOf(b))?.getTime() ?? 0),
|
||||
);
|
||||
const toFetch = [...fresh, ...refresh].slice(0, DETAIL_FETCH_CAP);
|
||||
console.log(
|
||||
`[ingest-grants] detail budget ${DETAIL_FETCH_CAP}: ${Math.min(fresh.length, DETAIL_FETCH_CAP)} new, ${Math.max(0, Math.min(DETAIL_FETCH_CAP - fresh.length, refresh.length))} refresh, backlog remaining ${Math.max(0, fresh.length - DETAIL_FETCH_CAP)}`,
|
||||
|
||||
@@ -21,3 +21,11 @@ Nightly `matchGrants` workflow (05:15 UTC, after embeddings) — the plan's "SQL
|
||||
- Rolling (null) deadlines now PASS the runway gate and score 2/5 runway.
|
||||
- Candidate orgs exclude NTEE `T*` grantmakers, and matches self-gate by funder EIN plus normalized-name fallback — the first precedent run's top "leads" were foundations matched to themselves (NHDOJ registers grantmakers as charities; several lack resolved EINs).
|
||||
- First full run: 747 NH foundations, 21 batches (~6.5GB processed, 1 deferred), 2,766+ grants-paid rows, 123 synthesized foundation grants → **89 easy wins across 27 orgs**, top hero 69/100 with real matches like AIDS Response-Seacoast → Foundation for Seacoast Health. Coverage grows nightly as enrichment drains the org backlog (56 candidate orgs of ~6.2K NH registrants so far).
|
||||
|
||||
## v3 (2026-07-16): federal precedent via USASpending
|
||||
|
||||
Funder precedent now covers federal grants at the **program (ALN/CFDA) level** — the criterion was always valid for federal funders; only the data was missing. Ingest captures each opportunity's ALN numbers (`grants.alns`); the nightly `federalPrecedent` workflow (04:45) queries USASpending's award search (free, official, no key) for grant awards to NH recipients over a 3-year lookback per distinct program, stamping `program_state_award_count` + sample recipients onto every open grant carrying that ALN (multi-ALN grants keep the highest count). Match retrieval feeds it through the same `funderStateGrantCount` input and 25-point tiers foundations use. The match detail page shows the recipients table ("Recent program awards to NH recipients") with an explicit caution: distinguish one incumbent's renewals from genuine spread across orgs.
|
||||
|
||||
Ingest refresh also now rotates oldest-verified-first (`serverMapGrantVerification`) — the plain-Set version re-fetched the same head of the search results every pass, which had left 365 of 565 grants without ALNs.
|
||||
|
||||
First sweep: 156 distinct programs, 85 with NH history, 468/564 open federal grants carrying precedent; first federal easy-wins appeared (score 66, precedent 25/25, real Aug-24 deadline).
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE "grants" ADD COLUMN "alns" text[];--> statement-breakpoint
|
||||
ALTER TABLE "grants" ADD COLUMN "program_state_award_count" integer;--> statement-breakpoint
|
||||
ALTER TABLE "grants" ADD COLUMN "program_state_awards" jsonb;
|
||||
1478
packages/outreach-core/drizzle/server/meta/1784257140_snapshot.json
Normal file
1478
packages/outreach-core/drizzle/server/meta/1784257140_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,13 @@
|
||||
"when": 1784236406321,
|
||||
"tag": "1784236406_funder-precedent-index",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 4,
|
||||
"version": "7",
|
||||
"when": 1784257140142,
|
||||
"tag": "1784257140_federal-precedent",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -143,6 +143,16 @@ export const grants = pgTable(
|
||||
// Links 990-PF-synthesized grants to their foundation for the
|
||||
// precedent subscore; null for public-RFP sources.
|
||||
funderEin: text('funder_ein'),
|
||||
// Federal Assistance Listing Numbers (CFDA), e.g. ['93.243'] — the
|
||||
// join key into USASpending award history for federal precedent.
|
||||
alns: text('alns').array(),
|
||||
// Denormalized federal precedent: recent awards this grant's
|
||||
// program(s) made to recipients in the target state (USASpending),
|
||||
// refreshed by the federalPrecedent workflow. Null for non-federal
|
||||
// sources and never touched by ingest upserts.
|
||||
programStateAwardCount: integer('program_state_award_count'),
|
||||
// Sample recipients backing the count — review-page evidence.
|
||||
programStateAwards: jsonb('program_state_awards'),
|
||||
status: grantStatusEnum('status').notNull().default('open'),
|
||||
synopsisEmbedding: vector('synopsis_embedding', { dimensions: 1536 }),
|
||||
lastVerifiedAt: timestamp('last_verified_at', { withTimezone: true }),
|
||||
|
||||
@@ -2,3 +2,4 @@ export * from './expire-closed-grants.server.js';
|
||||
export * from './insert-grants.server.js';
|
||||
export * from './set-grant-embeddings.server.js';
|
||||
export * from './close-grants-for-funders.server.js';
|
||||
export * from './set-federal-precedent.server.js';
|
||||
|
||||
@@ -47,6 +47,8 @@ export async function serverInsertGrants(
|
||||
applicationEffortEstimate: sql`excluded.application_effort_estimate`,
|
||||
applicationFormSupported: sql`excluded.application_form_supported`,
|
||||
source: sql`excluded.source`,
|
||||
alns: sql`excluded.alns`,
|
||||
funderEin: sql`excluded.funder_ein`,
|
||||
status: sql`excluded.status`,
|
||||
lastVerifiedAt: sql`excluded.last_verified_at`,
|
||||
updatedAt: sql`now()`,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { and, eq, sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
export interface AlnPrecedent {
|
||||
readonly aln: string;
|
||||
readonly awardCount: number;
|
||||
readonly samples: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies per-program USASpending precedent onto every open federal grant
|
||||
* carrying that ALN. Grants with multiple ALNs keep the HIGHEST count
|
||||
* seen (a grant reachable through any strongly-NH program inherits that
|
||||
* program's precedent), and samples follow whichever count won.
|
||||
*/
|
||||
export async function serverSetFederalPrecedent(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
precedent: AlnPrecedent,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(schema.grants)
|
||||
.set({
|
||||
programStateAwardCount: sql`GREATEST(COALESCE(${schema.grants.programStateAwardCount}, 0), ${precedent.awardCount})`,
|
||||
programStateAwards: sql`CASE
|
||||
WHEN COALESCE(${schema.grants.programStateAwardCount}, 0) <= ${precedent.awardCount}
|
||||
THEN ${JSON.stringify(precedent.samples)}::jsonb
|
||||
ELSE ${schema.grants.programStateAwards}
|
||||
END`,
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(schema.grants.source, 'grants_gov'),
|
||||
eq(schema.grants.status, 'open'),
|
||||
sql`${precedent.aln} = ANY(${schema.grants.alns})`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Zeroes precedent before a refresh pass so removed programs don't keep stale counts. */
|
||||
export async function serverResetFederalPrecedent(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(schema.grants)
|
||||
.set({ programStateAwardCount: null, programStateAwards: null })
|
||||
.where(eq(schema.grants.source, 'grants_gov'));
|
||||
}
|
||||
@@ -2,3 +2,4 @@ export * from './list-open-grants.server.js';
|
||||
export * from './list-grants-needing-embedding.server.js';
|
||||
export * from './list-eligible-grants-for-org.server.js';
|
||||
export * from './list-grant-source-urls.server.js';
|
||||
export * from './list-federal-alns.server.js';
|
||||
|
||||
@@ -62,12 +62,15 @@ export async function serverListEligibleGrantsForOrg(
|
||||
applicationFormSupported: schema.grants.applicationFormSupported,
|
||||
funderEin: schema.grants.funderEin,
|
||||
similarity: sql<number>`1 - (${schema.grants.synopsisEmbedding} <=> ${vector}::vector)`,
|
||||
funderStateGrantCount: sql<number | null>`(
|
||||
SELECT count(*)::int FROM funder_grants fg
|
||||
JOIN funders f ON fg.funder_id = f.id
|
||||
WHERE f.ein = ${schema.grants.funderEin}
|
||||
AND fg.recipient_state = ${filters.orgState}
|
||||
)`,
|
||||
funderStateGrantCount: sql<number | null>`CASE
|
||||
WHEN ${schema.grants.funderEin} IS NOT NULL THEN (
|
||||
SELECT count(*)::int FROM funder_grants fg
|
||||
JOIN funders f ON fg.funder_id = f.id
|
||||
WHERE f.ein = ${schema.grants.funderEin}
|
||||
AND fg.recipient_state = ${filters.orgState}
|
||||
)
|
||||
ELSE ${schema.grants.programStateAwardCount}
|
||||
END`,
|
||||
})
|
||||
.from(schema.grants)
|
||||
.where(
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import { and, eq, isNotNull, sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
/**
|
||||
* Distinct ALN (CFDA) numbers across open federal grants — the work list
|
||||
* for the federalPrecedent workflow. A few hundred programs cover the
|
||||
* whole corpus, so precedent is fetched per program, not per grant.
|
||||
*/
|
||||
export async function serverListOpenFederalAlns(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
): Promise<string[]> {
|
||||
const rows = await db
|
||||
.select({ aln: sql<string>`DISTINCT unnest(${schema.grants.alns})` })
|
||||
.from(schema.grants)
|
||||
.where(
|
||||
and(
|
||||
eq(schema.grants.source, 'grants_gov'),
|
||||
eq(schema.grants.status, 'open'),
|
||||
isNotNull(schema.grants.alns),
|
||||
),
|
||||
);
|
||||
return rows.map((r) => r.aln).sort();
|
||||
}
|
||||
@@ -4,17 +4,30 @@ import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
/**
|
||||
* All source URLs already ingested for one source — lets ingestion spend
|
||||
* its per-run detail-fetch budget on NEW opportunities first instead of
|
||||
* re-fetching the same head of the search results every night.
|
||||
* Source URL → lastVerifiedAt for everything already ingested from one
|
||||
* source. Ingestion spends its per-run detail budget on NEW opportunities
|
||||
* first, then refreshes KNOWN ones oldest-verified-first — a plain Set
|
||||
* caused the refresh half to re-fetch the same head of the search results
|
||||
* every pass.
|
||||
*/
|
||||
export async function serverMapGrantVerification(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
source: (typeof schema.grants.$inferSelect)['source'],
|
||||
): Promise<Map<string, Date | null>> {
|
||||
const rows = await db
|
||||
.select({
|
||||
sourceUrl: schema.grants.sourceUrl,
|
||||
lastVerifiedAt: schema.grants.lastVerifiedAt,
|
||||
})
|
||||
.from(schema.grants)
|
||||
.where(eq(schema.grants.source, source));
|
||||
return new Map(rows.map((r) => [r.sourceUrl, r.lastVerifiedAt]));
|
||||
}
|
||||
|
||||
/** @deprecated superseded by serverMapGrantVerification; kept for callers needing only membership. */
|
||||
export async function serverListGrantSourceUrls(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
source: (typeof schema.grants.$inferSelect)['source'],
|
||||
): Promise<Set<string>> {
|
||||
const rows = await db
|
||||
.select({ sourceUrl: schema.grants.sourceUrl })
|
||||
.from(schema.grants)
|
||||
.where(eq(schema.grants.source, source));
|
||||
return new Set(rows.map((r) => r.sourceUrl));
|
||||
return new Set((await serverMapGrantVerification(db, source)).keys());
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ export async function serverGetMatchDetail(
|
||||
grantEligibility: schema.grants.eligibilityEntityTypes,
|
||||
grantGeo: schema.grants.geographicScope,
|
||||
grantEffort: schema.grants.applicationEffortEstimate,
|
||||
grantProgramAwards: schema.grants.programStateAwards,
|
||||
})
|
||||
.from(schema.matches)
|
||||
.innerJoin(schema.orgs, eq(schema.matches.orgId, schema.orgs.id))
|
||||
@@ -109,6 +110,23 @@ export async function serverGetMatchDetail(
|
||||
|
||||
let funderGivingHistory: MatchDetail['funderGivingHistory'] = [];
|
||||
let funderApplicationInfo: unknown = null;
|
||||
if (row.grantFunderEin == null && Array.isArray(row.grantProgramAwards)) {
|
||||
// Federal grants: USASpending program awards fill the same evidence
|
||||
// table (recipient/amount/year) the 990-PF history uses.
|
||||
funderGivingHistory = (
|
||||
row.grantProgramAwards as Array<{
|
||||
recipientName?: string;
|
||||
amount?: number | null;
|
||||
startDate?: string | null;
|
||||
}>
|
||||
).map((a) => ({
|
||||
recipientName: a.recipientName ?? 'Unknown recipient',
|
||||
recipientCity: null,
|
||||
amount: a.amount ?? null,
|
||||
purpose: null,
|
||||
taxYear: a.startDate != null ? Number(a.startDate.slice(0, 4)) : 0,
|
||||
}));
|
||||
}
|
||||
if (row.grantFunderEin != null) {
|
||||
const funderRow = await db
|
||||
.select({ applicationInfo: schema.funders.applicationInfo })
|
||||
|
||||
Reference in New Issue
Block a user