feat(scoring): 990-PF funder-precedent index — the 25-point subscore goes live
New funders/funder_grants schema + ingest990pf monthly workflow: IRS BMF state file discovers NH private foundations (747), e-file index CSVs select their latest 990-PF filings, batch ZIPs stream through fflate (4/run cap, most-hits-first, deferred logged), grants-paid rows land in funder_grants, and funders with >=2 NH grants synthesize rolling grant rows (source irs_990pf, funder_ein linked) that flow through the existing embed+match pipeline. Scoring v2: funderPrecedentSubscore tiers repeated in-state giving (1/3/5/10 -> 8/15/20/25); easy win = >=65 total AND >=12 precedent (plan's precedent floor); scale is the full 0-100. Rolling deadlines pass the runway gate. Retrieval computes per-funder in-state counts and exposes funder_ein. Lead-quality gates from the first precedent run's failures: candidate orgs exclude NTEE T* grantmakers; self-matches gated by EIN + normalized name (NHDOJ registers foundations as charities, several without resolved EINs — the first run's top 'leads' were foundations matched to themselves). Live: ~6.5GB of IRS batches processed, 2,766 grants-paid rows, 123 synthesized foundation grants, 89 easy wins across 27 orgs, credible top-10 (AIDS Response-Seacoast -> Foundation for Seacoast Health, 25/25 precedent). 153 tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
export * from './upsert-funder.server.js';
|
||||
export * from './replace-funder-grants.server.js';
|
||||
@@ -0,0 +1,48 @@
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
export interface FunderGrantInput {
|
||||
readonly recipientName: string;
|
||||
readonly recipientCity: string | null;
|
||||
readonly recipientState: string | null;
|
||||
readonly amount: number | null;
|
||||
readonly purpose: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces a funder's grants-paid rows for one tax year — re-parsing the
|
||||
* same filing is idempotent (990-PF rows have no stable per-grant id, so
|
||||
* per-year replace beats per-row upsert).
|
||||
*/
|
||||
export async function serverReplaceFunderGrantsForYear(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
funderId: string,
|
||||
taxYear: number,
|
||||
grants: ReadonlyArray<FunderGrantInput>,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.delete(schema.funderGrants)
|
||||
.where(
|
||||
and(
|
||||
eq(schema.funderGrants.funderId, funderId),
|
||||
eq(schema.funderGrants.taxYear, taxYear),
|
||||
),
|
||||
);
|
||||
|
||||
for (let i = 0; i < grants.length; i += 500) {
|
||||
const chunk = grants.slice(i, i + 500);
|
||||
await db.insert(schema.funderGrants).values(
|
||||
chunk.map((grant) => ({
|
||||
funderId,
|
||||
taxYear,
|
||||
recipientName: grant.recipientName,
|
||||
recipientCity: grant.recipientCity,
|
||||
recipientState: grant.recipientState,
|
||||
amount: grant.amount,
|
||||
purpose: grant.purpose,
|
||||
})),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
export type NewFunderInput = Omit<
|
||||
typeof schema.funders.$inferInsert,
|
||||
'id' | 'createdAt' | 'updatedAt'
|
||||
>;
|
||||
|
||||
/** Upserts a private foundation, keyed on EIN (BMF re-scans refresh in place). */
|
||||
export async function serverUpsertFunder(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
funder: NewFunderInput,
|
||||
): Promise<string> {
|
||||
const [row] = await db
|
||||
.insert(schema.funders)
|
||||
.values(funder)
|
||||
.onConflictDoUpdate({
|
||||
target: schema.funders.ein,
|
||||
set: {
|
||||
name: sql`excluded.name`,
|
||||
city: sql`excluded.city`,
|
||||
state: sql`excluded.state`,
|
||||
nteeCode: sql`excluded.ntee_code`,
|
||||
totalAssets: sql`excluded.total_assets`,
|
||||
// Filing-derived fields only advance when the incoming parse is
|
||||
// newer (or first): re-running discovery with null filing fields
|
||||
// must not wipe an earlier XML parse.
|
||||
applicationInfo: sql`COALESCE(excluded.application_info, funders.application_info)`,
|
||||
latestTaxYear: sql`GREATEST(COALESCE(excluded.latest_tax_year, 0), COALESCE(funders.latest_tax_year, 0))`,
|
||||
latestObjectId: sql`CASE WHEN COALESCE(excluded.latest_tax_year, 0) >= COALESCE(funders.latest_tax_year, 0) AND excluded.latest_object_id IS NOT NULL THEN excluded.latest_object_id ELSE funders.latest_object_id END`,
|
||||
updatedAt: sql`now()`,
|
||||
},
|
||||
})
|
||||
.returning({ id: schema.funders.id });
|
||||
|
||||
if (row == null) {
|
||||
throw new Error('serverUpsertFunder: upsert returned no row');
|
||||
}
|
||||
return row.id;
|
||||
}
|
||||
2
packages/outreach-core/src/funders/index.server.ts
Normal file
2
packages/outreach-core/src/funders/index.server.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export * from './actions/index.server.js';
|
||||
export * from './queries/index.server.js';
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './list-funder-synthesis-data.server.js';
|
||||
export * from './list-funder-latest-object-ids.server.js';
|
||||
@@ -0,0 +1,34 @@
|
||||
import { inArray } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
export interface FunderLatestObjectIdRow {
|
||||
readonly ein: string;
|
||||
readonly latestObjectId: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks up the `latestObjectId` already recorded for a set of funder EINs —
|
||||
* used by the 990-PF ingestion workflow to skip re-downloading/re-parsing a
|
||||
* batch ZIP entry whose filing it has already parsed (the e-file index's
|
||||
* selected filing for an EIN this run has the same object id as last run).
|
||||
* Returns only rows that already exist as funders; EINs with no funder row
|
||||
* yet are simply absent from the result (never re-processed unnecessarily).
|
||||
*/
|
||||
export async function serverListFunderLatestObjectIds(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
eins: ReadonlyArray<string>,
|
||||
): Promise<FunderLatestObjectIdRow[]> {
|
||||
if (eins.length === 0) return [];
|
||||
|
||||
const rows = await db
|
||||
.select({
|
||||
ein: schema.funders.ein,
|
||||
latestObjectId: schema.funders.latestObjectId,
|
||||
})
|
||||
.from(schema.funders)
|
||||
.where(inArray(schema.funders.ein, [...eins]));
|
||||
|
||||
return rows;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
|
||||
export interface FunderSynthesisRow {
|
||||
funderId: string;
|
||||
ein: string;
|
||||
name: string;
|
||||
city: string | null;
|
||||
state: string | null;
|
||||
applicationInfo: unknown;
|
||||
latestTaxYear: number | null;
|
||||
stateGrantCount: number;
|
||||
medianAmount: number | null;
|
||||
maxAmount: number | null;
|
||||
/** Up to 12 distinct purpose strings from in-state grants, longest first. */
|
||||
purposes: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Funders with enough in-state giving history to synthesize a grant row
|
||||
* from ("this foundation funds orgs like you") — the 990-PF play for
|
||||
* foundations with no public RFP. Aggregated per funder over grants paid
|
||||
* into `recipientState`.
|
||||
*/
|
||||
export async function serverListFunderSynthesisData(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
{
|
||||
recipientState,
|
||||
minStateGrants,
|
||||
}: { recipientState: string; minStateGrants: number },
|
||||
): Promise<FunderSynthesisRow[]> {
|
||||
const result = await db.execute(sql`
|
||||
SELECT
|
||||
f.id AS funder_id,
|
||||
f.ein,
|
||||
f.name,
|
||||
f.city,
|
||||
f.state,
|
||||
f.application_info,
|
||||
f.latest_tax_year,
|
||||
count(fg.id)::int AS state_grant_count,
|
||||
percentile_cont(0.5) WITHIN GROUP (ORDER BY fg.amount)
|
||||
FILTER (WHERE fg.amount IS NOT NULL) AS median_amount,
|
||||
max(fg.amount) AS max_amount,
|
||||
(
|
||||
SELECT array_agg(p.purpose)
|
||||
FROM (
|
||||
SELECT DISTINCT fg2.purpose
|
||||
FROM funder_grants fg2
|
||||
WHERE fg2.funder_id = f.id
|
||||
AND fg2.recipient_state = ${recipientState}
|
||||
AND fg2.purpose IS NOT NULL
|
||||
AND length(fg2.purpose) > 3
|
||||
ORDER BY fg2.purpose
|
||||
LIMIT 12
|
||||
) p
|
||||
) AS purposes
|
||||
FROM funders f
|
||||
JOIN funder_grants fg ON fg.funder_id = f.id
|
||||
WHERE fg.recipient_state = ${recipientState}
|
||||
GROUP BY f.id
|
||||
HAVING count(fg.id) >= ${minStateGrants}
|
||||
ORDER BY count(fg.id) DESC
|
||||
`);
|
||||
|
||||
const { rows } = result as unknown as {
|
||||
rows: Record<string, unknown>[];
|
||||
};
|
||||
return rows.map((r) => ({
|
||||
funderId: r.funder_id as string,
|
||||
ein: r.ein as string,
|
||||
name: r.name as string,
|
||||
city: (r.city as string) ?? null,
|
||||
state: (r.state as string) ?? null,
|
||||
applicationInfo: r.application_info ?? null,
|
||||
latestTaxYear: (r.latest_tax_year as number) ?? null,
|
||||
stateGrantCount: r.state_grant_count as number,
|
||||
medianAmount: r.median_amount == null ? null : Math.round(Number(r.median_amount)),
|
||||
maxAmount: r.max_amount == null ? null : Number(r.max_amount),
|
||||
purposes: (r.purposes as string[]) ?? [],
|
||||
}));
|
||||
}
|
||||
Reference in New Issue
Block a user