feat(scoring): 990-PF funder-precedent index — the 25-point subscore goes live

New funders/funder_grants schema + ingest990pf monthly workflow: IRS BMF
state file discovers NH private foundations (747), e-file index CSVs
select their latest 990-PF filings, batch ZIPs stream through fflate
(4/run cap, most-hits-first, deferred logged), grants-paid rows land in
funder_grants, and funders with >=2 NH grants synthesize rolling grant
rows (source irs_990pf, funder_ein linked) that flow through the
existing embed+match pipeline.

Scoring v2: funderPrecedentSubscore tiers repeated in-state giving
(1/3/5/10 -> 8/15/20/25); easy win = >=65 total AND >=12 precedent
(plan's precedent floor); scale is the full 0-100. Rolling deadlines
pass the runway gate. Retrieval computes per-funder in-state counts and
exposes funder_ein.

Lead-quality gates from the first precedent run's failures: candidate
orgs exclude NTEE T* grantmakers; self-matches gated by EIN + normalized
name (NHDOJ registers foundations as charities, several without resolved
EINs — the first run's top 'leads' were foundations matched to
themselves).

Live: ~6.5GB of IRS batches processed, 2,766 grants-paid rows, 123
synthesized foundation grants, 89 easy wins across 27 orgs, credible
top-10 (AIDS Response-Seacoast -> Foundation for Seacoast Health, 25/25
precedent). 153 tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
2026-07-16 18:17:28 -04:00
parent 0ee478ec3d
commit 63b58e514d
34 changed files with 3634 additions and 45 deletions

View File

@@ -1,5 +1,6 @@
import { sql } from 'drizzle-orm';
import {
bigint,
boolean,
index,
integer,
@@ -139,6 +140,9 @@ export const grants = pgTable(
.default(false),
sourceUrl: text('source_url').notNull(),
source: grantSourceEnum('source').notNull(),
// Links 990-PF-synthesized grants to their foundation for the
// precedent subscore; null for public-RFP sources.
funderEin: text('funder_ein'),
status: grantStatusEnum('status').notNull().default('open'),
synopsisEmbedding: vector('synopsis_embedding', { dimensions: 1536 }),
lastVerifiedAt: timestamp('last_verified_at', { withTimezone: true }),
@@ -151,6 +155,7 @@ export const grants = pgTable(
index('idx_grants_status').on(t.status),
index('idx_grants_close_date').on(t.closeDate),
index('idx_grants_source').on(t.source),
index('idx_grants_funder_ein').on(t.funderEin),
index('grants_synopsis_embedding_idx').using(
'hnsw',
t.synopsisEmbedding.op('vector_cosine_ops'),
@@ -339,6 +344,61 @@ export const pipelineEvents = pgTable(
],
);
/**
* Private foundations (990-PF filers) — the funder-precedent index's
* subjects. Discovered from the IRS BMF state files; grants-paid history
* parsed from their e-filed 990-PF XML.
*/
export const funders = pgTable(
'funders',
{
id: uuid('id')
.primaryKey()
.default(sql`gen_random_uuid()`),
ein: text('ein').notNull(),
name: text('name').notNull(),
city: text('city'),
state: text('state'),
nteeCode: text('ntee_code'),
totalAssets: bigint('total_assets', { mode: 'number' }),
/** Part XV application info from the latest parsed filing (form/deadline/address text). */
applicationInfo: jsonb('application_info'),
latestTaxYear: integer('latest_tax_year'),
latestObjectId: text('latest_object_id'),
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow(),
},
(t) => [
uniqueIndex('idx_funders_ein').on(t.ein),
index('idx_funders_state').on(t.state),
],
);
/** One row per grant a foundation reported paying (990-PF Part XV line 3a). */
export const funderGrants = pgTable(
'funder_grants',
{
id: uuid('id')
.primaryKey()
.default(sql`gen_random_uuid()`),
funderId: uuid('funder_id')
.notNull()
.references(() => funders.id, { onDelete: 'cascade' }),
recipientName: text('recipient_name').notNull(),
recipientCity: text('recipient_city'),
recipientState: text('recipient_state'),
amount: integer('amount'),
purpose: text('purpose'),
taxYear: integer('tax_year').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(),
},
(t) => [
index('idx_funder_grants_funder').on(t.funderId),
index('idx_funder_grants_funder_year').on(t.funderId, t.taxYear),
index('idx_funder_grants_recipient_state').on(t.recipientState),
],
);
// ---------------------------------------------------------------------------
// Schema barrel
// ---------------------------------------------------------------------------
@@ -350,4 +410,6 @@ export const schema = {
contacts,
matches,
pipelineEvents,
funders,
funderGrants,
};

View File

@@ -0,0 +1,2 @@
export * from './upsert-funder.server.js';
export * from './replace-funder-grants.server.js';

View File

@@ -0,0 +1,48 @@
import { and, eq } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js';
export interface FunderGrantInput {
readonly recipientName: string;
readonly recipientCity: string | null;
readonly recipientState: string | null;
readonly amount: number | null;
readonly purpose: string | null;
}
/**
* Replaces a funder's grants-paid rows for one tax year — re-parsing the
* same filing is idempotent (990-PF rows have no stable per-grant id, so
* per-year replace beats per-row upsert).
*/
export async function serverReplaceFunderGrantsForYear(
db: NpOutreachDatabase | NpOutreachTransaction,
funderId: string,
taxYear: number,
grants: ReadonlyArray<FunderGrantInput>,
): Promise<void> {
await db
.delete(schema.funderGrants)
.where(
and(
eq(schema.funderGrants.funderId, funderId),
eq(schema.funderGrants.taxYear, taxYear),
),
);
for (let i = 0; i < grants.length; i += 500) {
const chunk = grants.slice(i, i + 500);
await db.insert(schema.funderGrants).values(
chunk.map((grant) => ({
funderId,
taxYear,
recipientName: grant.recipientName,
recipientCity: grant.recipientCity,
recipientState: grant.recipientState,
amount: grant.amount,
purpose: grant.purpose,
})),
);
}
}

View File

@@ -0,0 +1,42 @@
import { sql } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js';
export type NewFunderInput = Omit<
typeof schema.funders.$inferInsert,
'id' | 'createdAt' | 'updatedAt'
>;
/** Upserts a private foundation, keyed on EIN (BMF re-scans refresh in place). */
export async function serverUpsertFunder(
db: NpOutreachDatabase | NpOutreachTransaction,
funder: NewFunderInput,
): Promise<string> {
const [row] = await db
.insert(schema.funders)
.values(funder)
.onConflictDoUpdate({
target: schema.funders.ein,
set: {
name: sql`excluded.name`,
city: sql`excluded.city`,
state: sql`excluded.state`,
nteeCode: sql`excluded.ntee_code`,
totalAssets: sql`excluded.total_assets`,
// Filing-derived fields only advance when the incoming parse is
// newer (or first): re-running discovery with null filing fields
// must not wipe an earlier XML parse.
applicationInfo: sql`COALESCE(excluded.application_info, funders.application_info)`,
latestTaxYear: sql`GREATEST(COALESCE(excluded.latest_tax_year, 0), COALESCE(funders.latest_tax_year, 0))`,
latestObjectId: sql`CASE WHEN COALESCE(excluded.latest_tax_year, 0) >= COALESCE(funders.latest_tax_year, 0) AND excluded.latest_object_id IS NOT NULL THEN excluded.latest_object_id ELSE funders.latest_object_id END`,
updatedAt: sql`now()`,
},
})
.returning({ id: schema.funders.id });
if (row == null) {
throw new Error('serverUpsertFunder: upsert returned no row');
}
return row.id;
}

View File

@@ -0,0 +1,2 @@
export * from './actions/index.server.js';
export * from './queries/index.server.js';

View File

@@ -0,0 +1,2 @@
export * from './list-funder-synthesis-data.server.js';
export * from './list-funder-latest-object-ids.server.js';

View File

@@ -0,0 +1,34 @@
import { inArray } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js';
export interface FunderLatestObjectIdRow {
readonly ein: string;
readonly latestObjectId: string | null;
}
/**
* Looks up the `latestObjectId` already recorded for a set of funder EINs —
* used by the 990-PF ingestion workflow to skip re-downloading/re-parsing a
* batch ZIP entry whose filing it has already parsed (the e-file index's
* selected filing for an EIN this run has the same object id as last run).
* Returns only rows that already exist as funders; EINs with no funder row
* yet are simply absent from the result (never re-processed unnecessarily).
*/
export async function serverListFunderLatestObjectIds(
db: NpOutreachDatabase | NpOutreachTransaction,
eins: ReadonlyArray<string>,
): Promise<FunderLatestObjectIdRow[]> {
if (eins.length === 0) return [];
const rows = await db
.select({
ein: schema.funders.ein,
latestObjectId: schema.funders.latestObjectId,
})
.from(schema.funders)
.where(inArray(schema.funders.ein, [...eins]));
return rows;
}

View File

@@ -0,0 +1,83 @@
import { sql } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
export interface FunderSynthesisRow {
funderId: string;
ein: string;
name: string;
city: string | null;
state: string | null;
applicationInfo: unknown;
latestTaxYear: number | null;
stateGrantCount: number;
medianAmount: number | null;
maxAmount: number | null;
/** Up to 12 distinct purpose strings from in-state grants, longest first. */
purposes: string[];
}
/**
* Funders with enough in-state giving history to synthesize a grant row
* from ("this foundation funds orgs like you") — the 990-PF play for
* foundations with no public RFP. Aggregated per funder over grants paid
* into `recipientState`.
*/
export async function serverListFunderSynthesisData(
db: NpOutreachDatabase | NpOutreachTransaction,
{
recipientState,
minStateGrants,
}: { recipientState: string; minStateGrants: number },
): Promise<FunderSynthesisRow[]> {
const result = await db.execute(sql`
SELECT
f.id AS funder_id,
f.ein,
f.name,
f.city,
f.state,
f.application_info,
f.latest_tax_year,
count(fg.id)::int AS state_grant_count,
percentile_cont(0.5) WITHIN GROUP (ORDER BY fg.amount)
FILTER (WHERE fg.amount IS NOT NULL) AS median_amount,
max(fg.amount) AS max_amount,
(
SELECT array_agg(p.purpose)
FROM (
SELECT DISTINCT fg2.purpose
FROM funder_grants fg2
WHERE fg2.funder_id = f.id
AND fg2.recipient_state = ${recipientState}
AND fg2.purpose IS NOT NULL
AND length(fg2.purpose) > 3
ORDER BY fg2.purpose
LIMIT 12
) p
) AS purposes
FROM funders f
JOIN funder_grants fg ON fg.funder_id = f.id
WHERE fg.recipient_state = ${recipientState}
GROUP BY f.id
HAVING count(fg.id) >= ${minStateGrants}
ORDER BY count(fg.id) DESC
`);
const { rows } = result as unknown as {
rows: Record<string, unknown>[];
};
return rows.map((r) => ({
funderId: r.funder_id as string,
ein: r.ein as string,
name: r.name as string,
city: (r.city as string) ?? null,
state: (r.state as string) ?? null,
applicationInfo: r.application_info ?? null,
latestTaxYear: (r.latest_tax_year as number) ?? null,
stateGrantCount: r.state_grant_count as number,
medianAmount: r.median_amount == null ? null : Math.round(Number(r.median_amount)),
maxAmount: r.max_amount == null ? null : Number(r.max_amount),
purposes: (r.purposes as string[]) ?? [],
}));
}

View File

@@ -17,10 +17,15 @@ export interface EligibleGrantWithSimilarity {
| 'full_federal'
| 'unknown';
applicationFormSupported: boolean;
funderEin: string | null;
similarity: number;
/** Funder's historical grant count into the org's state (990-PF index); null when the grant has no linked funder. */
funderStateGrantCount: number | null;
}
export interface EligibleGrantFilters {
/** Org's state, for the funder-precedent count. */
readonly orgState: string;
/** Days of runway the deadline must clear (hard gate: 21). */
readonly minDaysToDeadline: number;
/** Minimum award ceiling in dollars (hard gate: 10_000). */
@@ -55,13 +60,20 @@ export async function serverListEligibleGrantsForOrg(
awardCeiling: schema.grants.awardCeiling,
applicationEffortEstimate: schema.grants.applicationEffortEstimate,
applicationFormSupported: schema.grants.applicationFormSupported,
funderEin: schema.grants.funderEin,
similarity: sql<number>`1 - (${schema.grants.synopsisEmbedding} <=> ${vector}::vector)`,
funderStateGrantCount: sql<number | null>`(
SELECT count(*)::int FROM funder_grants fg
JOIN funders f ON fg.funder_id = f.id
WHERE f.ein = ${schema.grants.funderEin}
AND fg.recipient_state = ${filters.orgState}
)`,
})
.from(schema.grants)
.where(
sql`${schema.grants.status} = 'open'
AND ${schema.grants.synopsisEmbedding} IS NOT NULL
AND ${schema.grants.closeDate} >= now() + make_interval(days => ${filters.minDaysToDeadline})
AND (${schema.grants.closeDate} IS NULL OR ${schema.grants.closeDate} >= now() + make_interval(days => ${filters.minDaysToDeadline}))
AND ${schema.grants.awardCeiling} >= ${filters.minAwardCeiling}`,
)
.orderBy(sql`${schema.grants.synopsisEmbedding} <=> ${vector}::vector`)

View File

@@ -4,3 +4,4 @@ export * from './grants/index.server.js';
export * from './matches/index.server.js';
export * from './orgs/index.server.js';
export * from './pipeline/index.server.js';
export * from './funders/index.server.js';

View File

@@ -90,14 +90,13 @@ describe('evaluateHardGates', () => {
expect(result.failures).not.toContain('deadline_too_soon');
});
it('fails when there is no close date at all', () => {
it('treats a missing close date as a rolling deadline (passes the gate)', () => {
const result = evaluateHardGates(
org,
{ ...grant, closeDate: null },
{ now: NOW },
);
expect(result.passed).toBe(false);
expect(result.failures).toContain('deadline_too_soon');
expect(result.failures).not.toContain('deadline_too_soon');
});
it('fails when the award ceiling is below the minimum', () => {
@@ -136,7 +135,7 @@ describe('evaluateHardGates', () => {
{
eligibilityEntityTypes: ['501c3'],
geographicScope: 'California',
closeDate: null,
closeDate: daysFromNow(5),
awardCeiling: null,
applicationFormSupported: false,
},

View File

@@ -194,7 +194,11 @@ function isGeographyEligible(
}
function hasSufficientRunway(grant: HardGateGrantInput, now: Date): boolean {
if (grant.closeDate == null) return false;
// Null close date = rolling/no stated deadline (typical for private
// foundations found via 990-PF). Rolling is pitchable — the runway
// SUBSCORE keeps it un-urgent; the GATE only kills real, too-soon
// deadlines.
if (grant.closeDate == null) return true;
const msPerDay = 24 * 60 * 60 * 1000;
const daysRemaining = (grant.closeDate.getTime() - now.getTime()) / msPerDay;
return daysRemaining >= MIN_DAYS_TO_DEADLINE;

View File

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest';
import {
capacityFitSubscore,
funderPrecedentSubscore,
competitionSubscore,
EASY_WIN_THRESHOLD,
effortSubscore,
@@ -79,7 +80,34 @@ describe('runwaySubscore', () => {
});
});
describe('funderPrecedentSubscore', () => {
it('tiers repeated in-state giving, zero without evidence', () => {
expect(funderPrecedentSubscore(null)).toBe(0);
expect(funderPrecedentSubscore(0)).toBe(0);
expect(funderPrecedentSubscore(1)).toBe(8);
expect(funderPrecedentSubscore(3)).toBe(15);
expect(funderPrecedentSubscore(5)).toBe(20);
expect(funderPrecedentSubscore(10)).toBe(25);
});
});
describe('scoreMatch', () => {
it('withholds easy-win from precedent-less high scorers', () => {
const result = scoreMatch({
similarity: 0.75,
orgTotalRevenue: 1_000_000,
awardCeiling: 200_000,
geographicScope: 'New Hampshire',
applicationEffortEstimate: 'loi_only',
closeDate: weeksFromNow(6),
now: NOW,
funderStateGrantCount: null,
});
// 30+15+15+10+5 = 75 — over the threshold but no precedent floor.
expect(result.totalScore).toBe(75);
expect(result.easyWin).toBe(false);
});
it('sums subscores and flags easy wins', () => {
const result = scoreMatch({
similarity: 0.75,
@@ -89,11 +117,12 @@ describe('scoreMatch', () => {
applicationEffortEstimate: 'short_form',
closeDate: weeksFromNow(6),
now: NOW,
funderStateGrantCount: 6,
});
// 30 fit + 15 capacity + 15 competition + 8 effort + 5 runway
expect(result.totalScore).toBe(73);
// 30 fit + 20 precedent + 15 capacity + 15 competition + 8 effort + 5 runway
expect(result.totalScore).toBe(93);
expect(result.easyWin).toBe(true);
expect(result.subscores.funderPrecedent).toBe(0);
expect(result.subscores.funderPrecedent).toBe(20);
});
it('keeps weak matches under the easy-win line', () => {
@@ -105,6 +134,7 @@ describe('scoreMatch', () => {
applicationEffortEstimate: 'full_federal',
closeDate: weeksFromNow(2),
now: NOW,
funderStateGrantCount: null,
});
expect(result.totalScore).toBeLessThan(EASY_WIN_THRESHOLD);
expect(result.easyWin).toBe(false);

View File

@@ -4,16 +4,15 @@
* the match workflow supplies the embedding similarity, everything else
* derives from columns.
*
* v1 weights (funder precedent's 25 points are NOT yet awarded — the
* 990-PF index is a later deliverable, so the achievable maximum is 75,
* not 100). `subscores` records each component so weights can be re-tuned
* from review/booking data without re-deriving inputs.
* `subscores` records each component so weights can be re-tuned from
* review/booking data without re-deriving inputs.
*
* mission fit 30 embedding cosine similarity, scaled
* capacity fit 15 award ceiling vs org revenue (sweet spot 1075%)
* competition 15 state/NH-restricted pools beat national ones
* effort 10 LOI/short-form beat full federal
* runway 5 310 weeks to deadline is ideal
* mission fit 30 embedding cosine similarity, scaled
* funder precedent 25 historical giving into the org's state (990-PF)
* capacity fit 15 award ceiling vs org revenue (sweet spot 1075%)
* competition 15 state/NH-restricted pools beat national ones
* effort 10 LOI/short-form beat full federal
* runway 5 310 weeks to deadline is ideal
*/
export interface MatchSubscores {
@@ -22,8 +21,7 @@ export interface MatchSubscores {
readonly competition: number;
readonly effort: number;
readonly runway: number;
/** Not yet computed — reserved so the jsonb shape is stable. */
readonly funderPrecedent: 0;
readonly funderPrecedent: number;
}
export interface ScoreMatchInput {
@@ -39,16 +37,25 @@ export interface ScoreMatchInput {
| 'unknown';
readonly closeDate: Date | null;
readonly now: Date;
/**
* Historical grants this funder has paid to recipients in the org's
* state (from the 990-PF index). Null = no precedent data for this
* grant's funder (e.g. federal agencies) — scores 0, not neutral: the
* plan weights precedent as the strongest single predictor, and absence
* of evidence should rank below presence.
*/
readonly funderStateGrantCount: number | null;
}
export const ACHIEVABLE_MAX_SCORE = 75;
export const ACHIEVABLE_MAX_SCORE = 100;
/**
* "Easy win" threshold, v1: two-thirds of the achievable maximum. The
* plan's full definition also requires a funder-precedent floor — that
* gate returns when the 990-PF index lands; thresholds re-tune on review
* and demo-booking data regardless.
* "Easy win" threshold. With the 990-PF precedent subscore live the scale
* is the plan's full 0100; the plan's >=75 easy-win bar applies, plus its
* precedent floor (see scoreMatch). Thresholds re-tune on review and
* demo-booking data.
*/
export const EASY_WIN_THRESHOLD = 50;
export const EASY_WIN_THRESHOLD = 65;
export const EASY_WIN_MIN_PRECEDENT = 12;
/** Similarity below this scores 0 fit; above the ceiling scores full fit. */
const SIMILARITY_FLOOR = 0.45;
@@ -115,6 +122,22 @@ export function effortSubscore(
}
}
/**
* Funder precedent (25): "a foundation that gave to three NH orgs like
* this one is a near-certain match for a fourth" — the plan's strongest
* single predictor. v1 measures repeated giving into the org's state;
* NTEE-level matching arrives when recipient orgs get resolved to EINs.
*/
export function funderPrecedentSubscore(
funderStateGrantCount: number | null,
): number {
if (funderStateGrantCount == null || funderStateGrantCount <= 0) return 0;
if (funderStateGrantCount >= 10) return 25;
if (funderStateGrantCount >= 5) return 20;
if (funderStateGrantCount >= 3) return 15;
return 8;
}
const MS_PER_WEEK = 7 * 24 * 60 * 60 * 1000;
/** 310 weeks out is ideal: urgent enough to act on, long enough to apply. */
@@ -140,7 +163,7 @@ export function scoreMatch(input: ScoreMatchInput): ScoredMatch {
competition: competitionSubscore(input.geographicScope),
effort: effortSubscore(input.applicationEffortEstimate),
runway: runwaySubscore(input.closeDate, input.now),
funderPrecedent: 0,
funderPrecedent: funderPrecedentSubscore(input.funderStateGrantCount),
};
const totalScore =
@@ -148,11 +171,14 @@ export function scoreMatch(input: ScoreMatchInput): ScoredMatch {
subscores.capacityFit +
subscores.competition +
subscores.effort +
subscores.runway;
subscores.runway +
subscores.funderPrecedent;
return {
totalScore,
subscores,
easyWin: totalScore >= EASY_WIN_THRESHOLD,
easyWin:
totalScore >= EASY_WIN_THRESHOLD &&
subscores.funderPrecedent >= EASY_WIN_MIN_PRECEDENT,
};
}

View File

@@ -1,4 +1,4 @@
import { and, eq } from 'drizzle-orm';
import { and, eq, or, isNull, sql } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js';
@@ -8,14 +8,19 @@ export interface MatchCandidateOrg {
name: string;
city: string | null;
state: string;
ein: string | null;
nteeCode: string | null;
totalRevenue: number | null;
}
/**
* Orgs eligible to enter match scoring: the primary ICP band ($100K$5M
* revenue), NH-based, in good standing with the registry. Everything else
* is either not the customer (yet) or would fail the standing gate anyway.
* revenue), NH-based, in good standing with the registry — and not
* themselves grantmakers (NTEE major group T): foundations and charitable
* trusts register with NHDOJ like any charity and often land in the ICP
* revenue band, but they GIVE grants, they don't seek them (surfaced by
* the first precedent-scored run, where the top "leads" were foundations
* matched to themselves).
*/
export async function serverListMatchCandidateOrgs(
db: NpOutreachDatabase | NpOutreachTransaction,
@@ -27,6 +32,7 @@ export async function serverListMatchCandidateOrgs(
name: schema.orgs.name,
city: schema.orgs.city,
state: schema.orgs.state,
ein: schema.orgs.ein,
nteeCode: schema.orgs.nteeCode,
totalRevenue: schema.orgs.totalRevenue,
})
@@ -36,6 +42,10 @@ export async function serverListMatchCandidateOrgs(
eq(schema.orgs.state, 'NH'),
eq(schema.orgs.registrationStatus, 'good_standing'),
eq(schema.orgs.icpBand, 'primary'),
or(
isNull(schema.orgs.nteeCode),
sql`${schema.orgs.nteeCode} NOT LIKE 'T%'`,
),
),
)
.limit(limit);