feat(scoring): match-generation workflow — scoring engine v1 (Stage 2, step 2)
matchGrants (nightly 05:15): per candidate org (NH + good standing + primary ICP) ensure a v0 NTEE-derived profile embedding (RETRIEVAL_QUERY, confidence 0.2 stub the Stage 4 profiler upgrades in place), retrieve top-50 open grants by pgvector cosine with SQL-enforced cheap gates (deadline >=21d, ceiling >=10K), run entity/geography gates in TS, score deterministically, upsert pair-keyed matches, reassign hero. Scoring: pure scoreMatch (mission fit 30 / capacity 15 / competition 15 / effort 10 / runway 5; precedent's 25 reserved until the 990-PF index; easy win >= 50/75). Eligibility gate now pattern-matches Grants.gov applicantTypes prose, conservatively (ambiguous entries do not admit). application_form_supported ignored for pass/fail per the manual-first decision, still recorded in rationale. Schema: unique (org_id, grant_id) on matches; unique org_id on org_profiles (latest-profile semantics). Review queue query now ordered hero > easy-win > score and capped at 100. Live run: 64 orgs -> 3,200 matches in 28s, 0 easy wins / max 39 — the honest result of an NIH-heavy 200-grant corpus vs NH service nonprofits; engine mechanics verified, corpus breadth is the next lever. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
import { eq, sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
/**
|
||||
* Recomputes the org's hero match: its single top-ranked non-rejected
|
||||
* match, easy wins first, then total score. Every org gets exactly one
|
||||
* hero (its Email 1 grant); runners-up stay ranked for Email 2.
|
||||
*/
|
||||
export async function serverAssignHeroMatch(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
orgId: string,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(schema.matches)
|
||||
.set({ isHero: false, updatedAt: sql`now()` })
|
||||
.where(eq(schema.matches.orgId, orgId));
|
||||
|
||||
await db.execute(sql`
|
||||
UPDATE matches SET is_hero = true, updated_at = now()
|
||||
WHERE id = (
|
||||
SELECT id FROM matches
|
||||
WHERE org_id = ${orgId}
|
||||
AND review_status != 'rejected'
|
||||
AND hard_gates_passed = true
|
||||
ORDER BY easy_win DESC, total_score DESC, created_at ASC
|
||||
LIMIT 1
|
||||
)
|
||||
`);
|
||||
}
|
||||
@@ -1,2 +1,4 @@
|
||||
export * from './insert-match.server.js';
|
||||
export * from './set-match-review.server.js';
|
||||
export * from './upsert-match-score.server.js';
|
||||
export * from './assign-hero-matches.server.js';
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
import type { MatchSubscores } from '../scoring.js';
|
||||
|
||||
export interface MatchScoreInput {
|
||||
readonly orgId: string;
|
||||
readonly grantId: string;
|
||||
readonly totalScore: number;
|
||||
readonly subscores: MatchSubscores;
|
||||
readonly hardGatesPassed: boolean;
|
||||
readonly easyWin: boolean;
|
||||
/** Gate failures, similarity, and any judge citations — audit trail. */
|
||||
readonly rationale: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a scored (org, grant) match, refreshing scores in place on
|
||||
* re-runs (unique on org+grant). Review fields are deliberately NOT
|
||||
* touched on conflict: a human's approve/reject stands even when the
|
||||
* nightly re-score moves the numbers — resurfacing rejected matches would
|
||||
* erode the review queue's trust, and re-approving approved ones is
|
||||
* pointless churn.
|
||||
*/
|
||||
export async function serverUpsertMatchScore(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
match: MatchScoreInput,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.insert(schema.matches)
|
||||
.values({
|
||||
orgId: match.orgId,
|
||||
grantId: match.grantId,
|
||||
totalScore: match.totalScore,
|
||||
subscores: match.subscores,
|
||||
hardGatesPassed: match.hardGatesPassed,
|
||||
easyWin: match.easyWin,
|
||||
rationale: match.rationale,
|
||||
reviewStatus: 'pending',
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [schema.matches.orgId, schema.matches.grantId],
|
||||
set: {
|
||||
totalScore: sql`excluded.total_score`,
|
||||
subscores: sql`excluded.subscores`,
|
||||
hardGatesPassed: sql`excluded.hard_gates_passed`,
|
||||
easyWin: sql`excluded.easy_win`,
|
||||
rationale: sql`excluded.rationale`,
|
||||
updatedAt: sql`now()`,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
entryAdmitsEntity,
|
||||
evaluateHardGates,
|
||||
type HardGateGrantInput,
|
||||
type HardGateOrgInput,
|
||||
@@ -151,3 +152,28 @@ describe('evaluateHardGates', () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('entryAdmitsEntity', () => {
|
||||
it('admits 501c3 orgs on Grants.gov prose entries', () => {
|
||||
expect(
|
||||
entryAdmitsEntity(
|
||||
'Nonprofits having a 501(c)(3) status with the IRS, other than institutions of higher education',
|
||||
'501c3',
|
||||
),
|
||||
).toBe(true);
|
||||
expect(entryAdmitsEntity('501c3', '501c3')).toBe(true);
|
||||
expect(entryAdmitsEntity('Nonprofit organizations', '501c3')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects negated and ambiguous entries', () => {
|
||||
expect(
|
||||
entryAdmitsEntity(
|
||||
'Nonprofits that do not have a 501(c)(3) status with the IRS, other than institutions of higher education',
|
||||
'501c3',
|
||||
),
|
||||
).toBe(false);
|
||||
expect(entryAdmitsEntity('Others (see text field entitled "Additional Information on Eligibility")', '501c3')).toBe(false);
|
||||
expect(entryAdmitsEntity('County governments', '501c3')).toBe(false);
|
||||
expect(entryAdmitsEntity('Nonprofits other than faith-based organizations', '501c3')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -128,6 +128,38 @@ export function evaluateHardGates(
|
||||
return { passed: failures.length === 0, failures };
|
||||
}
|
||||
|
||||
const P501C3_PATTERN = /501\s*\(?\s*c\s*\)?\s*\(?\s*3\s*\)?/i;
|
||||
/** "Nonprofits that do not have / without a 501(c)(3) status..." */
|
||||
const P501C3_NEGATED_PATTERN = /(?:do not have|without)\s+(?:a\s+)?501/i;
|
||||
const GENERIC_NEGATION_PATTERN = /other than|do not have|without|excluding/i;
|
||||
|
||||
/**
|
||||
* Does one eligibility-list entry admit this org? Entries arrive two ways:
|
||||
* short codes from curated sources ('501c3'), matched exactly, and prose
|
||||
* descriptions from Grants.gov applicantTypes ("Nonprofits having a
|
||||
* 501(c)(3) status with the IRS, other than institutions of higher
|
||||
* education"), matched by pattern. Deliberately conservative — an
|
||||
* ambiguous entry ("Others: see text field") does NOT admit; a wrongly
|
||||
* claimed eligibility in an outbound email is the failure mode this whole
|
||||
* engine exists to prevent.
|
||||
*/
|
||||
export function entryAdmitsEntity(entry: string, entityType: string): boolean {
|
||||
const e = entry.trim();
|
||||
if (e.toLowerCase() === entityType.toLowerCase()) return true;
|
||||
|
||||
if (entityType.toLowerCase() === '501c3') {
|
||||
// Entry names the 501(c)(3) code: admits unless it names it only to
|
||||
// exclude it ("nonprofits that do not have a 501(c)(3) status").
|
||||
if (P501C3_PATTERN.test(e)) return !P501C3_NEGATED_PATTERN.test(e);
|
||||
// Generic nonprofit entry with no 501-qualifier and no carve-out.
|
||||
if (/\bnon-?profits?\b/i.test(e) && !GENERIC_NEGATION_PATTERN.test(e)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function isEntityEligible(
|
||||
org: HardGateOrgInput,
|
||||
grant: HardGateGrantInput,
|
||||
@@ -138,8 +170,8 @@ function isEntityEligible(
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return grant.eligibilityEntityTypes.some(
|
||||
(entityType) => entityType.toLowerCase() === org.entityType.toLowerCase(),
|
||||
return grant.eligibilityEntityTypes.some((entry) =>
|
||||
entryAdmitsEntity(entry, org.entityType),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { desc, eq } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
@@ -37,5 +37,14 @@ export async function serverListPendingReviewMatches(
|
||||
.from(schema.matches)
|
||||
.innerJoin(schema.orgs, eq(schema.matches.orgId, schema.orgs.id))
|
||||
.innerJoin(schema.grants, eq(schema.matches.grantId, schema.grants.id))
|
||||
.where(eq(schema.matches.reviewStatus, 'pending'));
|
||||
.where(eq(schema.matches.reviewStatus, 'pending'))
|
||||
// Reviewers see the best candidates first: heroes, then easy wins,
|
||||
// then raw score. Capped — nightly re-scoring generates thousands of
|
||||
// pending pairs and the queue is worked top-down, not exhaustively.
|
||||
.orderBy(
|
||||
desc(schema.matches.isHero),
|
||||
desc(schema.matches.easyWin),
|
||||
desc(schema.matches.totalScore),
|
||||
)
|
||||
.limit(100);
|
||||
}
|
||||
|
||||
112
packages/outreach-core/src/matches/scoring.test.ts
Normal file
112
packages/outreach-core/src/matches/scoring.test.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
capacityFitSubscore,
|
||||
competitionSubscore,
|
||||
EASY_WIN_THRESHOLD,
|
||||
effortSubscore,
|
||||
missionFitSubscore,
|
||||
runwaySubscore,
|
||||
scoreMatch,
|
||||
} from './scoring.js';
|
||||
|
||||
const NOW = new Date('2026-07-16T00:00:00Z');
|
||||
|
||||
function weeksFromNow(weeks: number): Date {
|
||||
return new Date(NOW.getTime() + weeks * 7 * 24 * 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
describe('missionFitSubscore', () => {
|
||||
it('scales similarity between floor and ceiling to 0–30', () => {
|
||||
expect(missionFitSubscore(0.45)).toBe(0);
|
||||
expect(missionFitSubscore(0.6)).toBe(15);
|
||||
expect(missionFitSubscore(0.75)).toBe(30);
|
||||
});
|
||||
|
||||
it('clamps outside the band', () => {
|
||||
expect(missionFitSubscore(0.1)).toBe(0);
|
||||
expect(missionFitSubscore(0.95)).toBe(30);
|
||||
});
|
||||
});
|
||||
|
||||
describe('capacityFitSubscore', () => {
|
||||
it('gives full marks in the 10–75% sweet spot', () => {
|
||||
expect(capacityFitSubscore(100_000, 1_000_000)).toBe(15);
|
||||
expect(capacityFitSubscore(750_000, 1_000_000)).toBe(15);
|
||||
});
|
||||
|
||||
it('penalizes awards dwarfing the org', () => {
|
||||
expect(capacityFitSubscore(480_000, 150_000)).toBe(2);
|
||||
expect(capacityFitSubscore(1_200_000, 1_000_000)).toBe(8);
|
||||
});
|
||||
|
||||
it('penalizes trivially small awards', () => {
|
||||
expect(capacityFitSubscore(20_000, 1_000_000)).toBe(4);
|
||||
});
|
||||
|
||||
it('is neutral on unknown revenue, zero on unknown award', () => {
|
||||
expect(capacityFitSubscore(100_000, null)).toBe(7);
|
||||
expect(capacityFitSubscore(null, 1_000_000)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('competitionSubscore', () => {
|
||||
it('scores restricted pools high, national low', () => {
|
||||
expect(competitionSubscore('New Hampshire')).toBe(15);
|
||||
expect(competitionSubscore('Statewide - NH')).toBe(15);
|
||||
expect(competitionSubscore('New England')).toBe(10);
|
||||
expect(competitionSubscore(null)).toBe(3);
|
||||
expect(competitionSubscore('National')).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('effortSubscore', () => {
|
||||
it('orders loi > short form > unknown > full federal', () => {
|
||||
expect(effortSubscore('loi_only')).toBe(10);
|
||||
expect(effortSubscore('short_form')).toBe(8);
|
||||
expect(effortSubscore('unknown')).toBe(4);
|
||||
expect(effortSubscore('full_federal')).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runwaySubscore', () => {
|
||||
it('peaks in the 3–10 week window', () => {
|
||||
expect(runwaySubscore(weeksFromNow(2), NOW)).toBe(0);
|
||||
expect(runwaySubscore(weeksFromNow(5), NOW)).toBe(5);
|
||||
expect(runwaySubscore(weeksFromNow(15), NOW)).toBe(3);
|
||||
expect(runwaySubscore(weeksFromNow(30), NOW)).toBe(1);
|
||||
expect(runwaySubscore(null, NOW)).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoreMatch', () => {
|
||||
it('sums subscores and flags easy wins', () => {
|
||||
const result = scoreMatch({
|
||||
similarity: 0.75,
|
||||
orgTotalRevenue: 1_000_000,
|
||||
awardCeiling: 200_000,
|
||||
geographicScope: 'New Hampshire',
|
||||
applicationEffortEstimate: 'short_form',
|
||||
closeDate: weeksFromNow(6),
|
||||
now: NOW,
|
||||
});
|
||||
// 30 fit + 15 capacity + 15 competition + 8 effort + 5 runway
|
||||
expect(result.totalScore).toBe(73);
|
||||
expect(result.easyWin).toBe(true);
|
||||
expect(result.subscores.funderPrecedent).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps weak matches under the easy-win line', () => {
|
||||
const result = scoreMatch({
|
||||
similarity: 0.5,
|
||||
orgTotalRevenue: 150_000,
|
||||
awardCeiling: 480_000,
|
||||
geographicScope: null,
|
||||
applicationEffortEstimate: 'full_federal',
|
||||
closeDate: weeksFromNow(2),
|
||||
now: NOW,
|
||||
});
|
||||
expect(result.totalScore).toBeLessThan(EASY_WIN_THRESHOLD);
|
||||
expect(result.easyWin).toBe(false);
|
||||
});
|
||||
});
|
||||
158
packages/outreach-core/src/matches/scoring.ts
Normal file
158
packages/outreach-core/src/matches/scoring.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Deterministic weighted subscores for an (org, grant) pair — Stage 2 of
|
||||
* the scoring engine (docs/plan.md). Pure functions over plain inputs;
|
||||
* the match workflow supplies the embedding similarity, everything else
|
||||
* derives from columns.
|
||||
*
|
||||
* v1 weights (funder precedent's 25 points are NOT yet awarded — the
|
||||
* 990-PF index is a later deliverable, so the achievable maximum is 75,
|
||||
* not 100). `subscores` records each component so weights can be re-tuned
|
||||
* from review/booking data without re-deriving inputs.
|
||||
*
|
||||
* mission fit 30 embedding cosine similarity, scaled
|
||||
* capacity fit 15 award ceiling vs org revenue (sweet spot 10–75%)
|
||||
* competition 15 state/NH-restricted pools beat national ones
|
||||
* effort 10 LOI/short-form beat full federal
|
||||
* runway 5 3–10 weeks to deadline is ideal
|
||||
*/
|
||||
|
||||
export interface MatchSubscores {
|
||||
readonly missionFit: number;
|
||||
readonly capacityFit: number;
|
||||
readonly competition: number;
|
||||
readonly effort: number;
|
||||
readonly runway: number;
|
||||
/** Not yet computed — reserved so the jsonb shape is stable. */
|
||||
readonly funderPrecedent: 0;
|
||||
}
|
||||
|
||||
export interface ScoreMatchInput {
|
||||
/** Cosine similarity in [-1, 1] between org mission and grant synopsis. */
|
||||
readonly similarity: number;
|
||||
readonly orgTotalRevenue: number | null;
|
||||
readonly awardCeiling: number | null;
|
||||
readonly geographicScope: string | null;
|
||||
readonly applicationEffortEstimate:
|
||||
| 'loi_only'
|
||||
| 'short_form'
|
||||
| 'full_federal'
|
||||
| 'unknown';
|
||||
readonly closeDate: Date | null;
|
||||
readonly now: Date;
|
||||
}
|
||||
|
||||
export const ACHIEVABLE_MAX_SCORE = 75;
|
||||
/**
|
||||
* "Easy win" threshold, v1: two-thirds of the achievable maximum. The
|
||||
* plan's full definition also requires a funder-precedent floor — that
|
||||
* gate returns when the 990-PF index lands; thresholds re-tune on review
|
||||
* and demo-booking data regardless.
|
||||
*/
|
||||
export const EASY_WIN_THRESHOLD = 50;
|
||||
|
||||
/** Similarity below this scores 0 fit; above the ceiling scores full fit. */
|
||||
const SIMILARITY_FLOOR = 0.45;
|
||||
const SIMILARITY_CEILING = 0.75;
|
||||
|
||||
export function missionFitSubscore(similarity: number): number {
|
||||
const clamped = Math.min(
|
||||
Math.max(similarity, SIMILARITY_FLOOR),
|
||||
SIMILARITY_CEILING,
|
||||
);
|
||||
return Math.round(
|
||||
((clamped - SIMILARITY_FLOOR) / (SIMILARITY_CEILING - SIMILARITY_FLOOR)) * 30,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sweet spot: award is 10–75% of annual revenue (docs/plan.md). A grant
|
||||
* dwarfing the org's budget is a capacity red flag to federal funders; a
|
||||
* tiny one isn't worth the email. Unknown revenue scores a neutral 7.
|
||||
*/
|
||||
export function capacityFitSubscore(
|
||||
awardCeiling: number | null,
|
||||
orgTotalRevenue: number | null,
|
||||
): number {
|
||||
if (awardCeiling == null || orgTotalRevenue == null || orgTotalRevenue <= 0) {
|
||||
return awardCeiling == null ? 0 : 7;
|
||||
}
|
||||
const ratio = awardCeiling / orgTotalRevenue;
|
||||
if (ratio >= 0.1 && ratio <= 0.75) return 15;
|
||||
if (ratio >= 0.05 && ratio < 0.1) return 10;
|
||||
if (ratio > 0.75 && ratio <= 1.5) return 8;
|
||||
if (ratio < 0.05) return 4;
|
||||
return 2; // > 150% of revenue: real capacity red flag.
|
||||
}
|
||||
|
||||
const STATE_RESTRICTED_PATTERN =
|
||||
/new hampshire|\bnh\b|state of|statewide|county|municipal/i;
|
||||
const REGIONAL_PATTERN = /new england|northeast|regional/i;
|
||||
|
||||
/**
|
||||
* Competition proxy until expected-applicant-pool modeling exists:
|
||||
* geographically restricted pools are dramatically less competitive than
|
||||
* national ones. Null scope (typical for federal) = national = low score.
|
||||
*/
|
||||
export function competitionSubscore(geographicScope: string | null): number {
|
||||
if (geographicScope == null || geographicScope.trim() === '') return 3;
|
||||
if (STATE_RESTRICTED_PATTERN.test(geographicScope)) return 15;
|
||||
if (REGIONAL_PATTERN.test(geographicScope)) return 10;
|
||||
return 3;
|
||||
}
|
||||
|
||||
export function effortSubscore(
|
||||
estimate: ScoreMatchInput['applicationEffortEstimate'],
|
||||
): number {
|
||||
switch (estimate) {
|
||||
case 'loi_only':
|
||||
return 10;
|
||||
case 'short_form':
|
||||
return 8;
|
||||
case 'unknown':
|
||||
return 4;
|
||||
case 'full_federal':
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
const MS_PER_WEEK = 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
/** 3–10 weeks out is ideal: urgent enough to act on, long enough to apply. */
|
||||
export function runwaySubscore(closeDate: Date | null, now: Date): number {
|
||||
if (closeDate == null) return 2; // rolling/unknown deadline: usable, not urgent.
|
||||
const weeks = (closeDate.getTime() - now.getTime()) / MS_PER_WEEK;
|
||||
if (weeks < 3) return 0;
|
||||
if (weeks <= 10) return 5;
|
||||
if (weeks <= 20) return 3;
|
||||
return 1;
|
||||
}
|
||||
|
||||
export interface ScoredMatch {
|
||||
readonly totalScore: number;
|
||||
readonly subscores: MatchSubscores;
|
||||
readonly easyWin: boolean;
|
||||
}
|
||||
|
||||
export function scoreMatch(input: ScoreMatchInput): ScoredMatch {
|
||||
const subscores: MatchSubscores = {
|
||||
missionFit: missionFitSubscore(input.similarity),
|
||||
capacityFit: capacityFitSubscore(input.awardCeiling, input.orgTotalRevenue),
|
||||
competition: competitionSubscore(input.geographicScope),
|
||||
effort: effortSubscore(input.applicationEffortEstimate),
|
||||
runway: runwaySubscore(input.closeDate, input.now),
|
||||
funderPrecedent: 0,
|
||||
};
|
||||
|
||||
const totalScore =
|
||||
subscores.missionFit +
|
||||
subscores.capacityFit +
|
||||
subscores.competition +
|
||||
subscores.effort +
|
||||
subscores.runway;
|
||||
|
||||
return {
|
||||
totalScore,
|
||||
subscores,
|
||||
easyWin: totalScore >= EASY_WIN_THRESHOLD,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user