feat(scoring): federal funder precedent via USASpending (ALN-level)
Precedent was always a valid criterion for federal grants — the index just didn't cover them. Now: ingest captures ALN/CFDA numbers from fetchOpportunity (grants.alns); nightly federalPrecedent workflow (04:45) queries USASpending award search per distinct program (free official API, 3-year NH lookback) and stamps program_state_award_count + sample recipients onto open grants (multi-ALN keeps highest). Match retrieval feeds the same funderStateGrantCount input and 25-point tiers foundations use; detail page shows the recipients-evidence table with an incumbent-renewal caution. Also: detail refresh now rotates oldest-verified-first (serverMapGrantVerification) — the Set-based partition re-fetched the same 200 every pass, leaving 365/565 grants ALN-less. Live: 564/564 grants ALN-tagged, 156 programs swept, 85 with NH history, 468 grants carrying precedent, first federal easy-wins (66pts, 25/25 precedent, Aug-24 deadline). 158 tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -290,11 +290,15 @@ export default function MatchDetail({ loaderData }: Route.ComponentProps) {
|
|||||||
{nhHistory.length > 0 && (
|
{nhHistory.length > 0 && (
|
||||||
<section className="rounded border p-4">
|
<section className="rounded border p-4">
|
||||||
<h2 className="mb-1 text-lg font-semibold">
|
<h2 className="mb-1 text-lg font-semibold">
|
||||||
Funder giving history (from 990-PF filings)
|
{grant.source === 'irs_990pf'
|
||||||
|
? 'Funder giving history (from 990-PF filings)'
|
||||||
|
: 'Recent program awards to NH recipients (USASpending)'}
|
||||||
</h2>
|
</h2>
|
||||||
<p className="mb-3 text-sm text-gray-600">
|
<p className="mb-3 text-sm text-gray-600">
|
||||||
The concrete evidence behind the precedent score — who this
|
The concrete evidence behind the precedent score —{' '}
|
||||||
funder actually paid, sorted in-state first.
|
{grant.source === 'irs_990pf'
|
||||||
|
? 'who this funder actually paid, sorted in-state first.'
|
||||||
|
: 'who this federal program actually funded in NH recently. Watch for one incumbent recapturing renewals vs. genuine spread across orgs.'}
|
||||||
</p>
|
</p>
|
||||||
<table className="w-full text-sm">
|
<table className="w-full text-sm">
|
||||||
<thead>
|
<thead>
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ import pg from 'pg';
|
|||||||
import { setEmbedGrantsDeps } from './workflows/embed-grants.js';
|
import { setEmbedGrantsDeps } from './workflows/embed-grants.js';
|
||||||
import { setEnrichOrgsDeps } from './workflows/enrich-orgs.js';
|
import { setEnrichOrgsDeps } from './workflows/enrich-orgs.js';
|
||||||
import { setExpireGrantsDeps } from './workflows/expire-grants.js';
|
import { setExpireGrantsDeps } from './workflows/expire-grants.js';
|
||||||
|
import { setFederalPrecedentDeps } from './workflows/federal-precedent.js';
|
||||||
import { setIngestGrantsDeps } from './workflows/ingest-grants.js';
|
import { setIngestGrantsDeps } from './workflows/ingest-grants.js';
|
||||||
import { setIngest990pfDeps } from './workflows/ingest-990pf.js';
|
import { setIngest990pfDeps } from './workflows/ingest-990pf.js';
|
||||||
import { setIngestNhdojOrgsDeps } from './workflows/ingest-nhdoj-orgs.js';
|
import { setIngestNhdojOrgsDeps } from './workflows/ingest-nhdoj-orgs.js';
|
||||||
@@ -71,6 +72,7 @@ async function main() {
|
|||||||
setEmbedGrantsDeps({ db });
|
setEmbedGrantsDeps({ db });
|
||||||
setMatchGrantsDeps({ db });
|
setMatchGrantsDeps({ db });
|
||||||
setIngest990pfDeps({ db });
|
setIngest990pfDeps({ db });
|
||||||
|
setFederalPrecedentDeps({ db });
|
||||||
|
|
||||||
DBOS.setConfig({
|
DBOS.setConfig({
|
||||||
name: 'helmdocs-outreach-worker',
|
name: 'helmdocs-outreach-worker',
|
||||||
|
|||||||
@@ -28,6 +28,10 @@ import {
|
|||||||
runExpireGrantsNow,
|
runExpireGrantsNow,
|
||||||
setExpireGrantsDeps,
|
setExpireGrantsDeps,
|
||||||
} from './workflows/expire-grants.js';
|
} from './workflows/expire-grants.js';
|
||||||
|
import {
|
||||||
|
runFederalPrecedentNow,
|
||||||
|
setFederalPrecedentDeps,
|
||||||
|
} from './workflows/federal-precedent.js';
|
||||||
import {
|
import {
|
||||||
runIngestGrantsNow,
|
runIngestGrantsNow,
|
||||||
setIngestGrantsDeps,
|
setIngestGrantsDeps,
|
||||||
@@ -58,6 +62,7 @@ const RUNNERS: Record<string, () => Promise<void>> = {
|
|||||||
embedGrants: runEmbedGrantsNow,
|
embedGrants: runEmbedGrantsNow,
|
||||||
matchGrants: runMatchGrantsNow,
|
matchGrants: runMatchGrantsNow,
|
||||||
ingest990pf: runIngest990PfNow,
|
ingest990pf: runIngest990PfNow,
|
||||||
|
federalPrecedent: runFederalPrecedentNow,
|
||||||
};
|
};
|
||||||
|
|
||||||
const FIRST_RUN_ORDER = [
|
const FIRST_RUN_ORDER = [
|
||||||
@@ -67,6 +72,7 @@ const FIRST_RUN_ORDER = [
|
|||||||
'expireGrants',
|
'expireGrants',
|
||||||
'enrichOrgs',
|
'enrichOrgs',
|
||||||
'ingest990pf',
|
'ingest990pf',
|
||||||
|
'federalPrecedent',
|
||||||
'embedGrants',
|
'embedGrants',
|
||||||
'matchGrants',
|
'matchGrants',
|
||||||
];
|
];
|
||||||
@@ -106,6 +112,7 @@ async function main() {
|
|||||||
setEmbedGrantsDeps({ db });
|
setEmbedGrantsDeps({ db });
|
||||||
setMatchGrantsDeps({ db });
|
setMatchGrantsDeps({ db });
|
||||||
setIngest990pfDeps({ db });
|
setIngest990pfDeps({ db });
|
||||||
|
setFederalPrecedentDeps({ db });
|
||||||
|
|
||||||
DBOS.setConfig({
|
DBOS.setConfig({
|
||||||
name: 'helmdocs-outreach-worker',
|
name: 'helmdocs-outreach-worker',
|
||||||
|
|||||||
@@ -82,6 +82,11 @@ export interface GrantsGovOpportunityDetail {
|
|||||||
readonly agencyCode?: string | null;
|
readonly agencyCode?: string | null;
|
||||||
} | null;
|
} | null;
|
||||||
readonly synopsis: GrantsGovSynopsis | null;
|
readonly synopsis: GrantsGovSynopsis | null;
|
||||||
|
/** Assistance Listing (CFDA) entries, e.g. [{ cfdaNumber: '93.243' }]. */
|
||||||
|
readonly cfdas?: ReadonlyArray<{
|
||||||
|
readonly cfdaNumber?: string | null;
|
||||||
|
readonly programTitle?: string | null;
|
||||||
|
}> | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface GrantsGovDetailResponseEnvelope {
|
interface GrantsGovDetailResponseEnvelope {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
searchHitFixture,
|
searchHitFixture,
|
||||||
} from '#~/sources/grants-gov/fixtures.js';
|
} from '#~/sources/grants-gov/fixtures.js';
|
||||||
import {
|
import {
|
||||||
|
normalizeAlns,
|
||||||
normalizeApplicantTypes,
|
normalizeApplicantTypes,
|
||||||
normalizeGrantsGovOpportunity,
|
normalizeGrantsGovOpportunity,
|
||||||
parseCount,
|
parseCount,
|
||||||
@@ -232,3 +233,23 @@ describe('normalizeApplicantTypes', () => {
|
|||||||
expect(normalizeApplicantTypes([])).toBeNull();
|
expect(normalizeApplicantTypes([])).toBeNull();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('normalizeAlns', () => {
|
||||||
|
it('extracts distinct valid ALN numbers', () => {
|
||||||
|
expect(
|
||||||
|
normalizeAlns([
|
||||||
|
{ cfdaNumber: '93.243' },
|
||||||
|
{ cfdaNumber: ' 93.243 ' },
|
||||||
|
{ cfdaNumber: '16.888' },
|
||||||
|
{ cfdaNumber: 'garbage' },
|
||||||
|
{ cfdaNumber: null },
|
||||||
|
]),
|
||||||
|
).toEqual(['93.243', '16.888']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null for empty or missing lists', () => {
|
||||||
|
expect(normalizeAlns(null)).toBeNull();
|
||||||
|
expect(normalizeAlns([])).toBeNull();
|
||||||
|
expect(normalizeAlns([{ cfdaNumber: 'x' }])).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -98,6 +98,24 @@ export function stripHtml(html: string | null | undefined): string | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Extracts non-empty eligibility descriptions from `synopsis.applicantTypes`. */
|
/** Extracts non-empty eligibility descriptions from `synopsis.applicantTypes`. */
|
||||||
|
/** Distinct, trimmed ALN/CFDA numbers ('93.243') from the detail's cfdas list. */
|
||||||
|
export function normalizeAlns(
|
||||||
|
cfdas:
|
||||||
|
| ReadonlyArray<{ readonly cfdaNumber?: string | null }>
|
||||||
|
| null
|
||||||
|
| undefined,
|
||||||
|
): string[] | null {
|
||||||
|
if (cfdas == null) return null;
|
||||||
|
const nums = [
|
||||||
|
...new Set(
|
||||||
|
cfdas
|
||||||
|
.map((c) => c.cfdaNumber?.trim())
|
||||||
|
.filter((v): v is string => v != null && /^\d{2}\.\d{3}$/.test(v)),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
return nums.length > 0 ? nums : null;
|
||||||
|
}
|
||||||
|
|
||||||
export function normalizeApplicantTypes(
|
export function normalizeApplicantTypes(
|
||||||
applicantTypes: ReadonlyArray<GrantsGovApplicantType> | null | undefined,
|
applicantTypes: ReadonlyArray<GrantsGovApplicantType> | null | undefined,
|
||||||
): string[] | null {
|
): string[] | null {
|
||||||
@@ -162,6 +180,7 @@ export function normalizeGrantsGovOpportunity(
|
|||||||
title: detail.opportunityTitle ?? hit.title,
|
title: detail.opportunityTitle ?? hit.title,
|
||||||
funder: resolveFunder(hit, detail),
|
funder: resolveFunder(hit, detail),
|
||||||
synopsis: stripHtml(synopsis?.synopsisDesc),
|
synopsis: stripHtml(synopsis?.synopsisDesc),
|
||||||
|
alns: normalizeAlns(detail.cfdas),
|
||||||
eligibilityEntityTypes: normalizeApplicantTypes(
|
eligibilityEntityTypes: normalizeApplicantTypes(
|
||||||
synopsis?.applicantTypes,
|
synopsis?.applicantTypes,
|
||||||
),
|
),
|
||||||
|
|||||||
93
apps/outreach-worker/src/sources/usaspending/client.ts
Normal file
93
apps/outreach-worker/src/sources/usaspending/client.ts
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
/**
|
||||||
|
* USASpending.gov award-search client — the federal analog of the 990-PF
|
||||||
|
* grants-paid index. Official API, free, no key.
|
||||||
|
* Docs: https://api.usaspending.gov/docs/endpoints
|
||||||
|
*/
|
||||||
|
|
||||||
|
const SEARCH_URL =
|
||||||
|
'https://api.usaspending.gov/api/v2/search/spending_by_award/';
|
||||||
|
|
||||||
|
/** Grant-shaped assistance award type codes (block/formula/project/coop). */
|
||||||
|
const GRANT_AWARD_TYPE_CODES = ['02', '03', '04', '05'];
|
||||||
|
|
||||||
|
const DEFAULT_POLITENESS_DELAY_MS = 300;
|
||||||
|
|
||||||
|
export interface ProgramStateAwardSample {
|
||||||
|
readonly recipientName: string;
|
||||||
|
readonly amount: number | null;
|
||||||
|
readonly startDate: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ProgramStateAwards {
|
||||||
|
readonly aln: string;
|
||||||
|
readonly awardCount: number;
|
||||||
|
readonly samples: ProgramStateAwardSample[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UsaSpendingClientOptions {
|
||||||
|
readonly fetchImpl?: typeof fetch;
|
||||||
|
readonly politenessDelayMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function wait(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recent grant awards under one Assistance Listing to recipients in one
|
||||||
|
* state: total count plus a handful of sample recipients (review-page
|
||||||
|
* evidence — lets a human spot "one incumbent's renewals" vs "spread
|
||||||
|
* across orgs like ours").
|
||||||
|
*/
|
||||||
|
export async function fetchProgramStateAwards(
|
||||||
|
aln: string,
|
||||||
|
state: string,
|
||||||
|
{ startDate, endDate }: { startDate: string; endDate: string },
|
||||||
|
options: UsaSpendingClientOptions = {},
|
||||||
|
): Promise<ProgramStateAwards> {
|
||||||
|
const fetchImpl = options.fetchImpl ?? fetch;
|
||||||
|
|
||||||
|
const res = await fetchImpl(SEARCH_URL, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
filters: {
|
||||||
|
award_type_codes: GRANT_AWARD_TYPE_CODES,
|
||||||
|
recipient_locations: [{ country: 'USA', state }],
|
||||||
|
time_period: [{ start_date: startDate, end_date: endDate }],
|
||||||
|
program_numbers: [aln],
|
||||||
|
},
|
||||||
|
fields: ['Award ID', 'Recipient Name', 'Award Amount', 'Start Date'],
|
||||||
|
limit: 10,
|
||||||
|
page: 1,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(
|
||||||
|
`USASpending search failed for ALN ${aln} (${state}): ${res.status} ${res.statusText}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = (await res.json()) as {
|
||||||
|
results?: Array<{
|
||||||
|
'Recipient Name'?: string | null;
|
||||||
|
'Award Amount'?: number | null;
|
||||||
|
'Start Date'?: string | null;
|
||||||
|
}>;
|
||||||
|
page_metadata?: { total?: number | null };
|
||||||
|
};
|
||||||
|
|
||||||
|
await wait(options.politenessDelayMs ?? DEFAULT_POLITENESS_DELAY_MS);
|
||||||
|
|
||||||
|
const samples: ProgramStateAwardSample[] = (body.results ?? []).map((r) => ({
|
||||||
|
recipientName: r['Recipient Name'] ?? 'Unknown recipient',
|
||||||
|
amount: r['Award Amount'] == null ? null : Math.round(r['Award Amount']),
|
||||||
|
startDate: r['Start Date'] ?? null,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return {
|
||||||
|
aln,
|
||||||
|
awardCount: body.page_metadata?.total ?? samples.length,
|
||||||
|
samples,
|
||||||
|
};
|
||||||
|
}
|
||||||
194
apps/outreach-worker/src/workflows/federal-precedent.ts
Normal file
194
apps/outreach-worker/src/workflows/federal-precedent.ts
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
/**
|
||||||
|
* Nightly federal-precedent workflow — USASpending award history per
|
||||||
|
* Assistance Listing (ALN/CFDA), the federal analog of the 990-PF index.
|
||||||
|
*
|
||||||
|
* For each distinct ALN across open federal grants: count grant awards to
|
||||||
|
* recipients in the target state over the last ~3 fiscal years, stamp the
|
||||||
|
* count + sample recipients onto every open grant carrying that ALN.
|
||||||
|
* Match retrieval feeds the count into the same 25-point precedent
|
||||||
|
* subscore foundations use — "this program funded N NH orgs recently" and
|
||||||
|
* "this foundation funded N NH orgs recently" are the same signal.
|
||||||
|
*
|
||||||
|
* Runs at 04:45 — after ingest (03:00, which refreshes ALNs) and before
|
||||||
|
* matching (05:15). Registration follows `ingest-grants.ts` exactly.
|
||||||
|
*/
|
||||||
|
import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
|
||||||
|
import type { schema } from '@novelpad/outreach-core';
|
||||||
|
import {
|
||||||
|
serverListOpenFederalAlns,
|
||||||
|
serverResetFederalPrecedent,
|
||||||
|
serverSetFederalPrecedent,
|
||||||
|
} from '@novelpad/outreach-core/server';
|
||||||
|
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||||
|
|
||||||
|
import {
|
||||||
|
fetchProgramStateAwards,
|
||||||
|
type ProgramStateAwards,
|
||||||
|
} from '#~/sources/usaspending/client.js';
|
||||||
|
|
||||||
|
export type OutreachDb = NodePgDatabase<typeof schema>;
|
||||||
|
|
||||||
|
const RECIPIENT_STATE = 'NH';
|
||||||
|
/** ~3 fiscal years of history — recent enough to predict, long enough to see a pattern. */
|
||||||
|
const LOOKBACK_YEARS = 3;
|
||||||
|
|
||||||
|
export interface FederalPrecedentDeps {
|
||||||
|
readonly db: OutreachDb;
|
||||||
|
}
|
||||||
|
|
||||||
|
let registeredDeps: FederalPrecedentDeps | null = null;
|
||||||
|
|
||||||
|
export function setFederalPrecedentDeps(deps: FederalPrecedentDeps): void {
|
||||||
|
registeredDeps = deps;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFederalPrecedentDeps(): FederalPrecedentDeps {
|
||||||
|
if (registeredDeps == null) {
|
||||||
|
throw new Error(
|
||||||
|
'FederalPrecedentDeps not registered. Call setFederalPrecedentDeps() before DBOS.launch().',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return registeredDeps;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listAlns(db: OutreachDb): Promise<string[]> {
|
||||||
|
return serverListOpenFederalAlns(db);
|
||||||
|
}
|
||||||
|
const listAlnsStep = DBOS.registerStep(listAlns, {
|
||||||
|
name: 'listOpenFederalAlns',
|
||||||
|
retriesAllowed: true,
|
||||||
|
maxAttempts: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
async function fetchAlnAwards(
|
||||||
|
aln: string,
|
||||||
|
startDate: string,
|
||||||
|
endDate: string,
|
||||||
|
): Promise<ProgramStateAwards> {
|
||||||
|
return fetchProgramStateAwards(aln, RECIPIENT_STATE, {
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const fetchAlnAwardsStep = DBOS.registerStep(fetchAlnAwards, {
|
||||||
|
name: 'fetchAlnStateAwards',
|
||||||
|
retriesAllowed: true,
|
||||||
|
maxAttempts: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
async function applyPrecedent(
|
||||||
|
db: OutreachDb,
|
||||||
|
precedent: ProgramStateAwards,
|
||||||
|
): Promise<void> {
|
||||||
|
await serverSetFederalPrecedent(db, {
|
||||||
|
aln: precedent.aln,
|
||||||
|
awardCount: precedent.awardCount,
|
||||||
|
samples: precedent.samples,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const applyPrecedentStep = DBOS.registerStep(applyPrecedent, {
|
||||||
|
name: 'applyFederalPrecedent',
|
||||||
|
retriesAllowed: true,
|
||||||
|
maxAttempts: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
async function resetPrecedent(db: OutreachDb): Promise<void> {
|
||||||
|
await serverResetFederalPrecedent(db);
|
||||||
|
}
|
||||||
|
const resetPrecedentStep = DBOS.registerStep(resetPrecedent, {
|
||||||
|
name: 'resetFederalPrecedent',
|
||||||
|
retriesAllowed: true,
|
||||||
|
maxAttempts: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
async function runFederalPrecedent(): Promise<void> {
|
||||||
|
const { db } = getFederalPrecedentDeps();
|
||||||
|
|
||||||
|
const alns = await listAlnsStep(db);
|
||||||
|
if (alns.length === 0) {
|
||||||
|
console.log(
|
||||||
|
'[federal-precedent] no open federal grants carry ALNs yet (ingest refresh pending?)',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const end = new Date();
|
||||||
|
const start = new Date(end);
|
||||||
|
start.setFullYear(start.getFullYear() - LOOKBACK_YEARS);
|
||||||
|
const startDate = start.toISOString().slice(0, 10);
|
||||||
|
const endDate = end.toISOString().slice(0, 10);
|
||||||
|
|
||||||
|
await resetPrecedentStep(db);
|
||||||
|
|
||||||
|
let programsWithHistory = 0;
|
||||||
|
let failed = 0;
|
||||||
|
for (const aln of alns) {
|
||||||
|
try {
|
||||||
|
const precedent = await fetchAlnAwardsStep(aln, startDate, endDate);
|
||||||
|
if (precedent.awardCount > 0) {
|
||||||
|
await applyPrecedentStep(db, precedent);
|
||||||
|
programsWithHistory++;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
failed++;
|
||||||
|
console.error(`[federal-precedent] ALN ${aln} failed:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[federal-precedent] programs=${alns.length} withNhHistory=${programsWithHistory} failed=${failed}`,
|
||||||
|
);
|
||||||
|
if (failed > 0 && failed / alns.length > 0.2) {
|
||||||
|
throw new Error(
|
||||||
|
`[federal-precedent] systemic failure: ${failed}/${alns.length} programs failed`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const g = globalThis as unknown as {
|
||||||
|
__outreachFederalPrecedentRegistered?: boolean;
|
||||||
|
__outreachFederalPrecedentHandle?: (
|
||||||
|
scheduledTime: Date,
|
||||||
|
startedAt: Date,
|
||||||
|
) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!g.__outreachFederalPrecedentRegistered) {
|
||||||
|
g.__outreachFederalPrecedentRegistered = true;
|
||||||
|
|
||||||
|
const federalPrecedent = async (_scheduledTime: Date, _startedAt: Date) => {
|
||||||
|
try {
|
||||||
|
await runFederalPrecedent();
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[federal-precedent] pass failed:', err);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Must be registered as BOTH a workflow and a scheduled function,
|
||||||
|
// referencing the same function object — see ingest-grants.ts.
|
||||||
|
g.__outreachFederalPrecedentHandle = DBOS.registerWorkflow(federalPrecedent, {
|
||||||
|
name: 'federalPrecedent',
|
||||||
|
});
|
||||||
|
DBOS.registerScheduled(federalPrecedent, {
|
||||||
|
crontab: '45 4 * * *',
|
||||||
|
name: 'federalPrecedent',
|
||||||
|
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Starts one durable run of this workflow immediately through DBOS —
|
||||||
|
* the exact production path (workflow + checkpointed steps), used by
|
||||||
|
* `run-once.ts` for supervised/manual passes. Requires deps injected and
|
||||||
|
* `DBOS.launch()` completed.
|
||||||
|
*/
|
||||||
|
export function runFederalPrecedentNow(): Promise<void> {
|
||||||
|
const handle = g.__outreachFederalPrecedentHandle;
|
||||||
|
if (handle == null) {
|
||||||
|
throw new Error(
|
||||||
|
'federalPrecedent is not registered; was this module imported before DBOS.launch()?',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return handle(new Date(), new Date());
|
||||||
|
}
|
||||||
@@ -16,7 +16,7 @@ import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
|
|||||||
import type { schema } from '@novelpad/outreach-core';
|
import type { schema } from '@novelpad/outreach-core';
|
||||||
import {
|
import {
|
||||||
serverInsertGrants,
|
serverInsertGrants,
|
||||||
serverListGrantSourceUrls,
|
serverMapGrantVerification,
|
||||||
type NewGrantInput,
|
type NewGrantInput,
|
||||||
} from '@novelpad/outreach-core/server';
|
} from '@novelpad/outreach-core/server';
|
||||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||||
@@ -100,11 +100,19 @@ async function fetchGrantDetails(
|
|||||||
// hits.slice(0, cap) re-fetched the same head of the search results
|
// hits.slice(0, cap) re-fetched the same head of the search results
|
||||||
// every night and never drained the backlog. Known opportunities fill
|
// every night and never drained the backlog. Known opportunities fill
|
||||||
// any remaining budget (refreshing close dates / lastVerifiedAt).
|
// any remaining budget (refreshing close dates / lastVerifiedAt).
|
||||||
const known = await serverListGrantSourceUrls(db, 'grants_gov');
|
const known = await serverMapGrantVerification(db, 'grants_gov');
|
||||||
const isKnown = (hit: GrantsGovSearchHit) =>
|
const urlOf = (hit: GrantsGovSearchHit) =>
|
||||||
known.has(`https://www.grants.gov/search-results-detail/${hit.id}`);
|
`https://www.grants.gov/search-results-detail/${hit.id}`;
|
||||||
const fresh = hits.filter((h) => !isKnown(h));
|
const fresh = hits.filter((h) => !known.has(urlOf(h)));
|
||||||
const refresh = hits.filter(isKnown);
|
// Oldest-verified first so refreshes rotate through the whole corpus
|
||||||
|
// instead of re-fetching the same head of the search results.
|
||||||
|
const refresh = hits
|
||||||
|
.filter((h) => known.has(urlOf(h)))
|
||||||
|
.sort(
|
||||||
|
(a, b) =>
|
||||||
|
(known.get(urlOf(a))?.getTime() ?? 0) -
|
||||||
|
(known.get(urlOf(b))?.getTime() ?? 0),
|
||||||
|
);
|
||||||
const toFetch = [...fresh, ...refresh].slice(0, DETAIL_FETCH_CAP);
|
const toFetch = [...fresh, ...refresh].slice(0, DETAIL_FETCH_CAP);
|
||||||
console.log(
|
console.log(
|
||||||
`[ingest-grants] detail budget ${DETAIL_FETCH_CAP}: ${Math.min(fresh.length, DETAIL_FETCH_CAP)} new, ${Math.max(0, Math.min(DETAIL_FETCH_CAP - fresh.length, refresh.length))} refresh, backlog remaining ${Math.max(0, fresh.length - DETAIL_FETCH_CAP)}`,
|
`[ingest-grants] detail budget ${DETAIL_FETCH_CAP}: ${Math.min(fresh.length, DETAIL_FETCH_CAP)} new, ${Math.max(0, Math.min(DETAIL_FETCH_CAP - fresh.length, refresh.length))} refresh, backlog remaining ${Math.max(0, fresh.length - DETAIL_FETCH_CAP)}`,
|
||||||
|
|||||||
@@ -21,3 +21,11 @@ Nightly `matchGrants` workflow (05:15 UTC, after embeddings) — the plan's "SQL
|
|||||||
- Rolling (null) deadlines now PASS the runway gate and score 2/5 runway.
|
- Rolling (null) deadlines now PASS the runway gate and score 2/5 runway.
|
||||||
- Candidate orgs exclude NTEE `T*` grantmakers, and matches self-gate by funder EIN plus normalized-name fallback — the first precedent run's top "leads" were foundations matched to themselves (NHDOJ registers grantmakers as charities; several lack resolved EINs).
|
- Candidate orgs exclude NTEE `T*` grantmakers, and matches self-gate by funder EIN plus normalized-name fallback — the first precedent run's top "leads" were foundations matched to themselves (NHDOJ registers grantmakers as charities; several lack resolved EINs).
|
||||||
- First full run: 747 NH foundations, 21 batches (~6.5GB processed, 1 deferred), 2,766+ grants-paid rows, 123 synthesized foundation grants → **89 easy wins across 27 orgs**, top hero 69/100 with real matches like AIDS Response-Seacoast → Foundation for Seacoast Health. Coverage grows nightly as enrichment drains the org backlog (56 candidate orgs of ~6.2K NH registrants so far).
|
- First full run: 747 NH foundations, 21 batches (~6.5GB processed, 1 deferred), 2,766+ grants-paid rows, 123 synthesized foundation grants → **89 easy wins across 27 orgs**, top hero 69/100 with real matches like AIDS Response-Seacoast → Foundation for Seacoast Health. Coverage grows nightly as enrichment drains the org backlog (56 candidate orgs of ~6.2K NH registrants so far).
|
||||||
|
|
||||||
|
## v3 (2026-07-16): federal precedent via USASpending
|
||||||
|
|
||||||
|
Funder precedent now covers federal grants at the **program (ALN/CFDA) level** — the criterion was always valid for federal funders; only the data was missing. Ingest captures each opportunity's ALN numbers (`grants.alns`); the nightly `federalPrecedent` workflow (04:45) queries USASpending's award search (free, official, no key) for grant awards to NH recipients over a 3-year lookback per distinct program, stamping `program_state_award_count` + sample recipients onto every open grant carrying that ALN (multi-ALN grants keep the highest count). Match retrieval feeds it through the same `funderStateGrantCount` input and 25-point tiers foundations use. The match detail page shows the recipients table ("Recent program awards to NH recipients") with an explicit caution: distinguish one incumbent's renewals from genuine spread across orgs.
|
||||||
|
|
||||||
|
Ingest refresh also now rotates oldest-verified-first (`serverMapGrantVerification`) — the plain-Set version re-fetched the same head of the search results every pass, which had left 365 of 565 grants without ALNs.
|
||||||
|
|
||||||
|
First sweep: 156 distinct programs, 85 with NH history, 468/564 open federal grants carrying precedent; first federal easy-wins appeared (score 66, precedent 25/25, real Aug-24 deadline).
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE "grants" ADD COLUMN "alns" text[];--> statement-breakpoint
|
||||||
|
ALTER TABLE "grants" ADD COLUMN "program_state_award_count" integer;--> statement-breakpoint
|
||||||
|
ALTER TABLE "grants" ADD COLUMN "program_state_awards" jsonb;
|
||||||
1478
packages/outreach-core/drizzle/server/meta/1784257140_snapshot.json
Normal file
1478
packages/outreach-core/drizzle/server/meta/1784257140_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,13 @@
|
|||||||
"when": 1784236406321,
|
"when": 1784236406321,
|
||||||
"tag": "1784236406_funder-precedent-index",
|
"tag": "1784236406_funder-precedent-index",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 4,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1784257140142,
|
||||||
|
"tag": "1784257140_federal-precedent",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -143,6 +143,16 @@ export const grants = pgTable(
|
|||||||
// Links 990-PF-synthesized grants to their foundation for the
|
// Links 990-PF-synthesized grants to their foundation for the
|
||||||
// precedent subscore; null for public-RFP sources.
|
// precedent subscore; null for public-RFP sources.
|
||||||
funderEin: text('funder_ein'),
|
funderEin: text('funder_ein'),
|
||||||
|
// Federal Assistance Listing Numbers (CFDA), e.g. ['93.243'] — the
|
||||||
|
// join key into USASpending award history for federal precedent.
|
||||||
|
alns: text('alns').array(),
|
||||||
|
// Denormalized federal precedent: recent awards this grant's
|
||||||
|
// program(s) made to recipients in the target state (USASpending),
|
||||||
|
// refreshed by the federalPrecedent workflow. Null for non-federal
|
||||||
|
// sources and never touched by ingest upserts.
|
||||||
|
programStateAwardCount: integer('program_state_award_count'),
|
||||||
|
// Sample recipients backing the count — review-page evidence.
|
||||||
|
programStateAwards: jsonb('program_state_awards'),
|
||||||
status: grantStatusEnum('status').notNull().default('open'),
|
status: grantStatusEnum('status').notNull().default('open'),
|
||||||
synopsisEmbedding: vector('synopsis_embedding', { dimensions: 1536 }),
|
synopsisEmbedding: vector('synopsis_embedding', { dimensions: 1536 }),
|
||||||
lastVerifiedAt: timestamp('last_verified_at', { withTimezone: true }),
|
lastVerifiedAt: timestamp('last_verified_at', { withTimezone: true }),
|
||||||
|
|||||||
@@ -2,3 +2,4 @@ export * from './expire-closed-grants.server.js';
|
|||||||
export * from './insert-grants.server.js';
|
export * from './insert-grants.server.js';
|
||||||
export * from './set-grant-embeddings.server.js';
|
export * from './set-grant-embeddings.server.js';
|
||||||
export * from './close-grants-for-funders.server.js';
|
export * from './close-grants-for-funders.server.js';
|
||||||
|
export * from './set-federal-precedent.server.js';
|
||||||
|
|||||||
@@ -47,6 +47,8 @@ export async function serverInsertGrants(
|
|||||||
applicationEffortEstimate: sql`excluded.application_effort_estimate`,
|
applicationEffortEstimate: sql`excluded.application_effort_estimate`,
|
||||||
applicationFormSupported: sql`excluded.application_form_supported`,
|
applicationFormSupported: sql`excluded.application_form_supported`,
|
||||||
source: sql`excluded.source`,
|
source: sql`excluded.source`,
|
||||||
|
alns: sql`excluded.alns`,
|
||||||
|
funderEin: sql`excluded.funder_ein`,
|
||||||
status: sql`excluded.status`,
|
status: sql`excluded.status`,
|
||||||
lastVerifiedAt: sql`excluded.last_verified_at`,
|
lastVerifiedAt: sql`excluded.last_verified_at`,
|
||||||
updatedAt: sql`now()`,
|
updatedAt: sql`now()`,
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import { and, eq, sql } from 'drizzle-orm';
|
||||||
|
|
||||||
|
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||||
|
import { schema } from '#~/db/db.js';
|
||||||
|
|
||||||
|
export interface AlnPrecedent {
|
||||||
|
readonly aln: string;
|
||||||
|
readonly awardCount: number;
|
||||||
|
readonly samples: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies per-program USASpending precedent onto every open federal grant
|
||||||
|
* carrying that ALN. Grants with multiple ALNs keep the HIGHEST count
|
||||||
|
* seen (a grant reachable through any strongly-NH program inherits that
|
||||||
|
* program's precedent), and samples follow whichever count won.
|
||||||
|
*/
|
||||||
|
export async function serverSetFederalPrecedent(
|
||||||
|
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||||
|
precedent: AlnPrecedent,
|
||||||
|
): Promise<void> {
|
||||||
|
await db
|
||||||
|
.update(schema.grants)
|
||||||
|
.set({
|
||||||
|
programStateAwardCount: sql`GREATEST(COALESCE(${schema.grants.programStateAwardCount}, 0), ${precedent.awardCount})`,
|
||||||
|
programStateAwards: sql`CASE
|
||||||
|
WHEN COALESCE(${schema.grants.programStateAwardCount}, 0) <= ${precedent.awardCount}
|
||||||
|
THEN ${JSON.stringify(precedent.samples)}::jsonb
|
||||||
|
ELSE ${schema.grants.programStateAwards}
|
||||||
|
END`,
|
||||||
|
updatedAt: sql`now()`,
|
||||||
|
})
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(schema.grants.source, 'grants_gov'),
|
||||||
|
eq(schema.grants.status, 'open'),
|
||||||
|
sql`${precedent.aln} = ANY(${schema.grants.alns})`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Zeroes precedent before a refresh pass so removed programs don't keep stale counts. */
|
||||||
|
export async function serverResetFederalPrecedent(
|
||||||
|
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||||
|
): Promise<void> {
|
||||||
|
await db
|
||||||
|
.update(schema.grants)
|
||||||
|
.set({ programStateAwardCount: null, programStateAwards: null })
|
||||||
|
.where(eq(schema.grants.source, 'grants_gov'));
|
||||||
|
}
|
||||||
@@ -2,3 +2,4 @@ export * from './list-open-grants.server.js';
|
|||||||
export * from './list-grants-needing-embedding.server.js';
|
export * from './list-grants-needing-embedding.server.js';
|
||||||
export * from './list-eligible-grants-for-org.server.js';
|
export * from './list-eligible-grants-for-org.server.js';
|
||||||
export * from './list-grant-source-urls.server.js';
|
export * from './list-grant-source-urls.server.js';
|
||||||
|
export * from './list-federal-alns.server.js';
|
||||||
|
|||||||
@@ -62,12 +62,15 @@ export async function serverListEligibleGrantsForOrg(
|
|||||||
applicationFormSupported: schema.grants.applicationFormSupported,
|
applicationFormSupported: schema.grants.applicationFormSupported,
|
||||||
funderEin: schema.grants.funderEin,
|
funderEin: schema.grants.funderEin,
|
||||||
similarity: sql<number>`1 - (${schema.grants.synopsisEmbedding} <=> ${vector}::vector)`,
|
similarity: sql<number>`1 - (${schema.grants.synopsisEmbedding} <=> ${vector}::vector)`,
|
||||||
funderStateGrantCount: sql<number | null>`(
|
funderStateGrantCount: sql<number | null>`CASE
|
||||||
|
WHEN ${schema.grants.funderEin} IS NOT NULL THEN (
|
||||||
SELECT count(*)::int FROM funder_grants fg
|
SELECT count(*)::int FROM funder_grants fg
|
||||||
JOIN funders f ON fg.funder_id = f.id
|
JOIN funders f ON fg.funder_id = f.id
|
||||||
WHERE f.ein = ${schema.grants.funderEin}
|
WHERE f.ein = ${schema.grants.funderEin}
|
||||||
AND fg.recipient_state = ${filters.orgState}
|
AND fg.recipient_state = ${filters.orgState}
|
||||||
)`,
|
)
|
||||||
|
ELSE ${schema.grants.programStateAwardCount}
|
||||||
|
END`,
|
||||||
})
|
})
|
||||||
.from(schema.grants)
|
.from(schema.grants)
|
||||||
.where(
|
.where(
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import { and, eq, isNotNull, sql } from 'drizzle-orm';
|
||||||
|
|
||||||
|
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||||
|
import { schema } from '#~/db/db.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Distinct ALN (CFDA) numbers across open federal grants — the work list
|
||||||
|
* for the federalPrecedent workflow. A few hundred programs cover the
|
||||||
|
* whole corpus, so precedent is fetched per program, not per grant.
|
||||||
|
*/
|
||||||
|
export async function serverListOpenFederalAlns(
|
||||||
|
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const rows = await db
|
||||||
|
.select({ aln: sql<string>`DISTINCT unnest(${schema.grants.alns})` })
|
||||||
|
.from(schema.grants)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(schema.grants.source, 'grants_gov'),
|
||||||
|
eq(schema.grants.status, 'open'),
|
||||||
|
isNotNull(schema.grants.alns),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return rows.map((r) => r.aln).sort();
|
||||||
|
}
|
||||||
@@ -4,17 +4,30 @@ import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
|||||||
import { schema } from '#~/db/db.js';
|
import { schema } from '#~/db/db.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* All source URLs already ingested for one source — lets ingestion spend
|
* Source URL → lastVerifiedAt for everything already ingested from one
|
||||||
* its per-run detail-fetch budget on NEW opportunities first instead of
|
* source. Ingestion spends its per-run detail budget on NEW opportunities
|
||||||
* re-fetching the same head of the search results every night.
|
* first, then refreshes KNOWN ones oldest-verified-first — a plain Set
|
||||||
|
* caused the refresh half to re-fetch the same head of the search results
|
||||||
|
* every pass.
|
||||||
*/
|
*/
|
||||||
|
export async function serverMapGrantVerification(
|
||||||
|
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||||
|
source: (typeof schema.grants.$inferSelect)['source'],
|
||||||
|
): Promise<Map<string, Date | null>> {
|
||||||
|
const rows = await db
|
||||||
|
.select({
|
||||||
|
sourceUrl: schema.grants.sourceUrl,
|
||||||
|
lastVerifiedAt: schema.grants.lastVerifiedAt,
|
||||||
|
})
|
||||||
|
.from(schema.grants)
|
||||||
|
.where(eq(schema.grants.source, source));
|
||||||
|
return new Map(rows.map((r) => [r.sourceUrl, r.lastVerifiedAt]));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @deprecated superseded by serverMapGrantVerification; kept for callers needing only membership. */
|
||||||
export async function serverListGrantSourceUrls(
|
export async function serverListGrantSourceUrls(
|
||||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||||
source: (typeof schema.grants.$inferSelect)['source'],
|
source: (typeof schema.grants.$inferSelect)['source'],
|
||||||
): Promise<Set<string>> {
|
): Promise<Set<string>> {
|
||||||
const rows = await db
|
return new Set((await serverMapGrantVerification(db, source)).keys());
|
||||||
.select({ sourceUrl: schema.grants.sourceUrl })
|
|
||||||
.from(schema.grants)
|
|
||||||
.where(eq(schema.grants.source, source));
|
|
||||||
return new Set(rows.map((r) => r.sourceUrl));
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,6 +88,7 @@ export async function serverGetMatchDetail(
|
|||||||
grantEligibility: schema.grants.eligibilityEntityTypes,
|
grantEligibility: schema.grants.eligibilityEntityTypes,
|
||||||
grantGeo: schema.grants.geographicScope,
|
grantGeo: schema.grants.geographicScope,
|
||||||
grantEffort: schema.grants.applicationEffortEstimate,
|
grantEffort: schema.grants.applicationEffortEstimate,
|
||||||
|
grantProgramAwards: schema.grants.programStateAwards,
|
||||||
})
|
})
|
||||||
.from(schema.matches)
|
.from(schema.matches)
|
||||||
.innerJoin(schema.orgs, eq(schema.matches.orgId, schema.orgs.id))
|
.innerJoin(schema.orgs, eq(schema.matches.orgId, schema.orgs.id))
|
||||||
@@ -109,6 +110,23 @@ export async function serverGetMatchDetail(
|
|||||||
|
|
||||||
let funderGivingHistory: MatchDetail['funderGivingHistory'] = [];
|
let funderGivingHistory: MatchDetail['funderGivingHistory'] = [];
|
||||||
let funderApplicationInfo: unknown = null;
|
let funderApplicationInfo: unknown = null;
|
||||||
|
if (row.grantFunderEin == null && Array.isArray(row.grantProgramAwards)) {
|
||||||
|
// Federal grants: USASpending program awards fill the same evidence
|
||||||
|
// table (recipient/amount/year) the 990-PF history uses.
|
||||||
|
funderGivingHistory = (
|
||||||
|
row.grantProgramAwards as Array<{
|
||||||
|
recipientName?: string;
|
||||||
|
amount?: number | null;
|
||||||
|
startDate?: string | null;
|
||||||
|
}>
|
||||||
|
).map((a) => ({
|
||||||
|
recipientName: a.recipientName ?? 'Unknown recipient',
|
||||||
|
recipientCity: null,
|
||||||
|
amount: a.amount ?? null,
|
||||||
|
purpose: null,
|
||||||
|
taxYear: a.startDate != null ? Number(a.startDate.slice(0, 4)) : 0,
|
||||||
|
}));
|
||||||
|
}
|
||||||
if (row.grantFunderEin != null) {
|
if (row.grantFunderEin != null) {
|
||||||
const funderRow = await db
|
const funderRow = await db
|
||||||
.select({ applicationInfo: schema.funders.applicationInfo })
|
.select({ applicationInfo: schema.funders.applicationInfo })
|
||||||
|
|||||||
Reference in New Issue
Block a user