feat(scoring): 990-PF funder-precedent index — the 25-point subscore goes live

New funders/funder_grants schema + ingest990pf monthly workflow: IRS BMF
state file discovers NH private foundations (747), e-file index CSVs
select their latest 990-PF filings, batch ZIPs stream through fflate
(4/run cap, most-hits-first, deferred logged), grants-paid rows land in
funder_grants, and funders with >=2 NH grants synthesize rolling grant
rows (source irs_990pf, funder_ein linked) that flow through the
existing embed+match pipeline.

Scoring v2: funderPrecedentSubscore tiers repeated in-state giving
(1/3/5/10 -> 8/15/20/25); easy win = >=65 total AND >=12 precedent
(plan's precedent floor); scale is the full 0-100. Rolling deadlines
pass the runway gate. Retrieval computes per-funder in-state counts and
exposes funder_ein.

Lead-quality gates from the first precedent run's failures: candidate
orgs exclude NTEE T* grantmakers; self-matches gated by EIN + normalized
name (NHDOJ registers foundations as charities, several without resolved
EINs — the first run's top 'leads' were foundations matched to
themselves).

Live: ~6.5GB of IRS batches processed, 2,766 grants-paid rows, 123
synthesized foundation grants, 89 easy wins across 27 orgs, credible
top-10 (AIDS Response-Seacoast -> Foundation for Seacoast Health, 25/25
precedent). 153 tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
2026-07-16 18:17:28 -04:00
parent 0ee478ec3d
commit 63b58e514d
34 changed files with 3634 additions and 45 deletions

View File

@@ -0,0 +1,2 @@
export * from './upsert-funder.server.js';
export * from './replace-funder-grants.server.js';

View File

@@ -0,0 +1,48 @@
import { and, eq } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js';
export interface FunderGrantInput {
readonly recipientName: string;
readonly recipientCity: string | null;
readonly recipientState: string | null;
readonly amount: number | null;
readonly purpose: string | null;
}
/**
* Replaces a funder's grants-paid rows for one tax year — re-parsing the
* same filing is idempotent (990-PF rows have no stable per-grant id, so
* per-year replace beats per-row upsert).
*/
export async function serverReplaceFunderGrantsForYear(
db: NpOutreachDatabase | NpOutreachTransaction,
funderId: string,
taxYear: number,
grants: ReadonlyArray<FunderGrantInput>,
): Promise<void> {
await db
.delete(schema.funderGrants)
.where(
and(
eq(schema.funderGrants.funderId, funderId),
eq(schema.funderGrants.taxYear, taxYear),
),
);
for (let i = 0; i < grants.length; i += 500) {
const chunk = grants.slice(i, i + 500);
await db.insert(schema.funderGrants).values(
chunk.map((grant) => ({
funderId,
taxYear,
recipientName: grant.recipientName,
recipientCity: grant.recipientCity,
recipientState: grant.recipientState,
amount: grant.amount,
purpose: grant.purpose,
})),
);
}
}

View File

@@ -0,0 +1,42 @@
import { sql } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js';
export type NewFunderInput = Omit<
typeof schema.funders.$inferInsert,
'id' | 'createdAt' | 'updatedAt'
>;
/** Upserts a private foundation, keyed on EIN (BMF re-scans refresh in place). */
export async function serverUpsertFunder(
db: NpOutreachDatabase | NpOutreachTransaction,
funder: NewFunderInput,
): Promise<string> {
const [row] = await db
.insert(schema.funders)
.values(funder)
.onConflictDoUpdate({
target: schema.funders.ein,
set: {
name: sql`excluded.name`,
city: sql`excluded.city`,
state: sql`excluded.state`,
nteeCode: sql`excluded.ntee_code`,
totalAssets: sql`excluded.total_assets`,
// Filing-derived fields only advance when the incoming parse is
// newer (or first): re-running discovery with null filing fields
// must not wipe an earlier XML parse.
applicationInfo: sql`COALESCE(excluded.application_info, funders.application_info)`,
latestTaxYear: sql`GREATEST(COALESCE(excluded.latest_tax_year, 0), COALESCE(funders.latest_tax_year, 0))`,
latestObjectId: sql`CASE WHEN COALESCE(excluded.latest_tax_year, 0) >= COALESCE(funders.latest_tax_year, 0) AND excluded.latest_object_id IS NOT NULL THEN excluded.latest_object_id ELSE funders.latest_object_id END`,
updatedAt: sql`now()`,
},
})
.returning({ id: schema.funders.id });
if (row == null) {
throw new Error('serverUpsertFunder: upsert returned no row');
}
return row.id;
}

View File

@@ -0,0 +1,2 @@
export * from './actions/index.server.js';
export * from './queries/index.server.js';

View File

@@ -0,0 +1,2 @@
export * from './list-funder-synthesis-data.server.js';
export * from './list-funder-latest-object-ids.server.js';

View File

@@ -0,0 +1,34 @@
import { inArray } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
import { schema } from '#~/db/db.js';
export interface FunderLatestObjectIdRow {
readonly ein: string;
readonly latestObjectId: string | null;
}
/**
* Looks up the `latestObjectId` already recorded for a set of funder EINs —
* used by the 990-PF ingestion workflow to skip re-downloading/re-parsing a
* batch ZIP entry whose filing it has already parsed (the e-file index's
* selected filing for an EIN this run has the same object id as last run).
* Returns only rows that already exist as funders; EINs with no funder row
* yet are simply absent from the result (never re-processed unnecessarily).
*/
export async function serverListFunderLatestObjectIds(
db: NpOutreachDatabase | NpOutreachTransaction,
eins: ReadonlyArray<string>,
): Promise<FunderLatestObjectIdRow[]> {
if (eins.length === 0) return [];
const rows = await db
.select({
ein: schema.funders.ein,
latestObjectId: schema.funders.latestObjectId,
})
.from(schema.funders)
.where(inArray(schema.funders.ein, [...eins]));
return rows;
}

View File

@@ -0,0 +1,83 @@
import { sql } from 'drizzle-orm';
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
export interface FunderSynthesisRow {
funderId: string;
ein: string;
name: string;
city: string | null;
state: string | null;
applicationInfo: unknown;
latestTaxYear: number | null;
stateGrantCount: number;
medianAmount: number | null;
maxAmount: number | null;
/** Up to 12 distinct purpose strings from in-state grants, longest first. */
purposes: string[];
}
/**
* Funders with enough in-state giving history to synthesize a grant row
* from ("this foundation funds orgs like you") — the 990-PF play for
* foundations with no public RFP. Aggregated per funder over grants paid
* into `recipientState`.
*/
export async function serverListFunderSynthesisData(
db: NpOutreachDatabase | NpOutreachTransaction,
{
recipientState,
minStateGrants,
}: { recipientState: string; minStateGrants: number },
): Promise<FunderSynthesisRow[]> {
const result = await db.execute(sql`
SELECT
f.id AS funder_id,
f.ein,
f.name,
f.city,
f.state,
f.application_info,
f.latest_tax_year,
count(fg.id)::int AS state_grant_count,
percentile_cont(0.5) WITHIN GROUP (ORDER BY fg.amount)
FILTER (WHERE fg.amount IS NOT NULL) AS median_amount,
max(fg.amount) AS max_amount,
(
SELECT array_agg(p.purpose)
FROM (
SELECT DISTINCT fg2.purpose
FROM funder_grants fg2
WHERE fg2.funder_id = f.id
AND fg2.recipient_state = ${recipientState}
AND fg2.purpose IS NOT NULL
AND length(fg2.purpose) > 3
ORDER BY fg2.purpose
LIMIT 12
) p
) AS purposes
FROM funders f
JOIN funder_grants fg ON fg.funder_id = f.id
WHERE fg.recipient_state = ${recipientState}
GROUP BY f.id
HAVING count(fg.id) >= ${minStateGrants}
ORDER BY count(fg.id) DESC
`);
const { rows } = result as unknown as {
rows: Record<string, unknown>[];
};
return rows.map((r) => ({
funderId: r.funder_id as string,
ein: r.ein as string,
name: r.name as string,
city: (r.city as string) ?? null,
state: (r.state as string) ?? null,
applicationInfo: r.application_info ?? null,
latestTaxYear: (r.latest_tax_year as number) ?? null,
stateGrantCount: r.state_grant_count as number,
medianAmount: r.median_amount == null ? null : Math.round(Number(r.median_amount)),
maxAmount: r.max_amount == null ? null : Number(r.max_amount),
purposes: (r.purposes as string[]) ?? [],
}));
}