feat(scoring): v4 — peer precedent, mission-fit floor, LLM match judge
Fixes the federal mismatch class (boys' camp × NIH research center): - Peer precedent: federalPrecedent paginates USASpending (≤500 awards/ program) and name-matches every recipient against primary-ICP NH registry orgs (shared normalizeOrgNameForMatching, also used by the self-match gate). The 25-pt precedent tiers now key off program_state_peer_award_count — Dartmouth renewals and SBIR LLCs no longer grant precedent to community nonprofits. Raw count + peer- annotated award list stay as review evidence (peer badges, peers-first). - Mission-fit floor (12/30, grants_gov only): below it a match is stored with fit_viable=false and hidden from the pending queue, hero selection, and easy-win. Foundation-synthesized grants exempt (generic synopses). - Mission-fit judge live (judgeMatches, 06:15, 200/night best-first): JUDGE_MODEL reads the synopsis against the org profile with an explicit ignore-eligibility-breadth instruction; graded verdict with required citations; deterministic verdict→points map (27/18/8/0) sets missionFit, total, easy-win, and viability. Verdicts survive nightly re-scores via an upsert splice and re-enter the judge queue when the org profile is re-researched (org_profiles.updated_at). First sweep: 81/149 programs have NH history, only 6 have peer history; queue-head judging zeroes the research-mechanism garbage (mismatch) while surfacing genuine strong fits. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { buildMatchJudgePrompt } from './run.js';
|
||||
import {
|
||||
MatchJudgeVerdictSchema,
|
||||
missionFitFromVerdict,
|
||||
verdictIsFitViable,
|
||||
} from './schema.js';
|
||||
|
||||
describe('missionFitFromVerdict', () => {
|
||||
it('maps verdicts deterministically, capped under the embedding max', () => {
|
||||
expect(missionFitFromVerdict('strong_fit')).toBe(27);
|
||||
expect(missionFitFromVerdict('plausible')).toBe(18);
|
||||
expect(missionFitFromVerdict('weak')).toBe(8);
|
||||
expect(missionFitFromVerdict('mismatch')).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps only strong/plausible queue-viable', () => {
|
||||
expect(verdictIsFitViable('strong_fit')).toBe(true);
|
||||
expect(verdictIsFitViable('plausible')).toBe(true);
|
||||
expect(verdictIsFitViable('weak')).toBe(false);
|
||||
expect(verdictIsFitViable('mismatch')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MatchJudgeVerdictSchema', () => {
|
||||
it('requires grounding citations', () => {
|
||||
expect(() =>
|
||||
MatchJudgeVerdictSchema.parse({
|
||||
verdict: 'mismatch',
|
||||
citedOrgEvidence: '',
|
||||
citedGrantEvidence: 'x',
|
||||
reasoning: 'y',
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildMatchJudgePrompt', () => {
|
||||
const base = {
|
||||
org: {
|
||||
name: 'Camp Tecumseh',
|
||||
city: 'Moultonborough',
|
||||
missionStatement: 'Residential summer camp for boys',
|
||||
programs: [
|
||||
{ name: 'Summer camp', description: 'sports and outdoors', populationServed: 'boys 8-15' },
|
||||
],
|
||||
serviceGeography: 'Lakes Region NH',
|
||||
profileConfidence: 0.8,
|
||||
},
|
||||
grant: {
|
||||
title: 'Nutrition Obesity Research Centers (NORCs)',
|
||||
funder: 'NIH',
|
||||
synopsis: 'Supports research center infrastructure...',
|
||||
},
|
||||
};
|
||||
|
||||
it('instructs the judge to ignore eligibility breadth', () => {
|
||||
const prompt = buildMatchJudgePrompt(base);
|
||||
expect(prompt).toContain('IGNORE eligibility breadth');
|
||||
expect(prompt).toContain('Camp Tecumseh');
|
||||
expect(prompt).toContain('Nutrition Obesity Research Centers');
|
||||
expect(prompt).toContain('- Summer camp — sports and outdoors — serves boys 8-15');
|
||||
});
|
||||
|
||||
it('flags low-confidence stub profiles', () => {
|
||||
const prompt = buildMatchJudgePrompt({
|
||||
...base,
|
||||
org: { ...base.org, programs: [], profileConfidence: 0.2 },
|
||||
});
|
||||
expect(prompt).toContain('low-confidence');
|
||||
expect(prompt).toContain('no researched program list');
|
||||
});
|
||||
});
|
||||
@@ -1,64 +1,136 @@
|
||||
import { MissionFitVerdictSchema, type MissionFitVerdict } from './schema.js';
|
||||
import type { OrgProfile } from '../org-profiler/schema.js';
|
||||
import { getAi } from '../../gemini.js';
|
||||
import { JUDGE_MODEL } from '../../models.js';
|
||||
import {
|
||||
MATCH_JUDGE_VERDICT_JSON_SCHEMA,
|
||||
MatchJudgeVerdictSchema,
|
||||
type MatchJudgeVerdict,
|
||||
} from './schema.js';
|
||||
|
||||
export interface RunMissionFitJudgeInput {
|
||||
/** Extracted profile of the candidate org (from the Org Profiler). */
|
||||
orgProfile: OrgProfile;
|
||||
/** Grant program name / title, as scored against by the deterministic SQL gates. */
|
||||
grantProgramName: string;
|
||||
/** Funding priorities / eligible-use language pulled from the grant's own source text. */
|
||||
grantPriorities: string[];
|
||||
export interface JudgeOrgSide {
|
||||
readonly name: string;
|
||||
readonly city: string | null;
|
||||
/** Researched or NTEE-stub mission text (whatever the profile holds). */
|
||||
readonly missionStatement: string | null;
|
||||
/** Researched programs, if the Stage 4 profiler has run for this org. */
|
||||
readonly programs: Array<{
|
||||
name: string;
|
||||
description: string | null;
|
||||
populationServed: string | null;
|
||||
}>;
|
||||
readonly serviceGeography: string | null;
|
||||
/** Profile confidence — low means the org side is mostly an NTEE guess. */
|
||||
readonly profileConfidence: number | null;
|
||||
}
|
||||
|
||||
export interface JudgeGrantSide {
|
||||
readonly title: string;
|
||||
readonly funder: string;
|
||||
readonly synopsis: string | null;
|
||||
}
|
||||
|
||||
export interface RunMatchJudgeInput {
|
||||
readonly org: JudgeOrgSide;
|
||||
readonly grant: JudgeGrantSide;
|
||||
}
|
||||
|
||||
const MAX_SYNOPSIS_CHARS = 12_000;
|
||||
|
||||
/**
|
||||
* Build the judge prompt for one (org, grant) match candidate. Kept separate
|
||||
* from `runMissionFitJudge` so it's independently unit-testable once wired up.
|
||||
* Build the judge prompt for one (org, grant) candidate. Kept separate
|
||||
* from `runMatchJudge` so it's independently unit-testable.
|
||||
*
|
||||
* The core instruction exists because of a concrete failure mode:
|
||||
* federal (especially NIH) synopses declare near-universal *eligibility*
|
||||
* while the funded work is highly specific — "nonprofits may apply" put a
|
||||
* boys' summer camp on an obesity-research center grant. Eligibility
|
||||
* breadth is therefore explicitly out of scope; the judge reads what the
|
||||
* program FUNDS and who realistically performs that work.
|
||||
*/
|
||||
export function buildMissionFitJudgePrompt(input: RunMissionFitJudgeInput): string {
|
||||
export function buildMatchJudgePrompt(input: RunMatchJudgeInput): string {
|
||||
const { org, grant } = input;
|
||||
|
||||
const programLines =
|
||||
org.programs.length > 0
|
||||
? org.programs
|
||||
.map((p) =>
|
||||
[
|
||||
`- ${p.name}`,
|
||||
p.description,
|
||||
p.populationServed == null ? null : `serves ${p.populationServed}`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' — '),
|
||||
)
|
||||
.join('\n')
|
||||
: '(no researched program list — judge from the mission text alone)';
|
||||
|
||||
return [
|
||||
'You are the final mission-fit judge for a candidate (org, grant) match',
|
||||
'that has already passed deterministic SQL hard gates (eligibility,',
|
||||
'geography, award range). Decide whether the org\'s actual programs',
|
||||
'plausibly fit the grant\'s funding priorities. You MUST cite one',
|
||||
'concrete org program and one concrete grant priority your verdict is',
|
||||
'grounded in — a verdict without both citations is invalid. A wrong',
|
||||
'"fit: true" here can put a real NH nonprofit in front of a funder that',
|
||||
'will never fund them, so when the fit is unclear, prefer `fit: false`.',
|
||||
'You judge whether a specific nonprofit is a credible fit for a specific',
|
||||
'grant program. The pair already passed automated eligibility, geography,',
|
||||
'and deadline gates; your ONLY question is programmatic mission fit:',
|
||||
'does the work this org actually does match what this grant actually funds?',
|
||||
'',
|
||||
`Grant program: ${input.grantProgramName}`,
|
||||
`Grant priorities: ${input.grantPriorities.join('; ')}`,
|
||||
`Org legal name: ${input.orgProfile.legalName.value}`,
|
||||
`Org mission: ${input.orgProfile.mission.value}`,
|
||||
`Org program areas: ${input.orgProfile.programAreas.value.join('; ')}`,
|
||||
'Rules:',
|
||||
'- IGNORE eligibility breadth entirely. Federal synopses often say any',
|
||||
' nonprofit may apply while the funded activity is narrow, technical, or',
|
||||
' institutional (research centers, clinical trials, training programs at',
|
||||
' universities). Judge what gets FUNDED and who realistically performs',
|
||||
' that work, not who is allowed to apply.',
|
||||
'- A research-mechanism grant (center grants, clinical trials, R-series/',
|
||||
' P-series/U-series NIH mechanisms) fits only orgs that conduct that kind',
|
||||
' of research.',
|
||||
'- Ground the verdict in one concrete org activity and one concrete piece',
|
||||
' of synopsis language; a verdict without both citations is invalid.',
|
||||
"- A wrong positive verdict wastes a real fundraiser's time and burns our",
|
||||
' credibility with them. When genuinely torn between two verdicts, pick',
|
||||
' the lower one.',
|
||||
'',
|
||||
'Verdicts:',
|
||||
"- strong_fit: the org's core work is squarely what the program funds.",
|
||||
'- plausible: real overlap; a competent grant writer could make the case.',
|
||||
'- weak: tangential overlap only; the org would be an outlier applicant.',
|
||||
'- mismatch: the org does not do what this program funds.',
|
||||
'',
|
||||
'--- GRANT ---',
|
||||
`Title: ${grant.title}`,
|
||||
`Funder: ${grant.funder}`,
|
||||
`Synopsis: ${(grant.synopsis ?? '(none)').slice(0, MAX_SYNOPSIS_CHARS)}`,
|
||||
'',
|
||||
'--- ORGANIZATION ---',
|
||||
`Name: ${org.name}`,
|
||||
`Location: ${org.city ?? 'unknown'}, NH`,
|
||||
`Mission: ${org.missionStatement ?? '(unknown)'}`,
|
||||
`Service area: ${org.serviceGeography ?? '(unknown)'}`,
|
||||
'Programs:',
|
||||
programLines,
|
||||
...(org.profileConfidence != null && org.profileConfidence < 0.5
|
||||
? [
|
||||
'',
|
||||
'NOTE: this org profile is low-confidence (category-derived, not',
|
||||
'researched). Judge from the mission category; do not invent programs.',
|
||||
]
|
||||
: []),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* NOT IMPLEMENTED — this judge is the veto gate before a match can reach a
|
||||
* human reviewer (and, downstream, a real prospect via Apollo), so it should
|
||||
* not go live against real matches until the deterministic hard-gate scoring
|
||||
* this package doesn't own is wired in ahead of it. Intended production call
|
||||
* shape, mirroring novelpad-desktop's
|
||||
* packages/ai/src/agents/grant/section-drafter/run.ts (invokeVertex +
|
||||
* responseSchema-constrained structured JSON output) — note `JUDGE_MODEL`,
|
||||
* not `BULK_MODEL`: a wrong verdict here reaches a prospect:
|
||||
*
|
||||
* import { getAi } from '../../gemini.js';
|
||||
* import { JUDGE_MODEL } from '../../models.js';
|
||||
*
|
||||
* const result = await getAi().models.generateContent({
|
||||
* model: JUDGE_MODEL,
|
||||
* contents: buildMissionFitJudgePrompt(input),
|
||||
* config: {
|
||||
* responseMimeType: 'application/json',
|
||||
* // responseSchema: MISSION_FIT_VERDICT_RESPONSE_SCHEMA — a Type/Schema
|
||||
* // literal from '@google/genai' hand-mirroring MissionFitVerdictSchema.
|
||||
* temperature: 0.1,
|
||||
* },
|
||||
* });
|
||||
* const raw = JSON.parse(result.text ?? '{}');
|
||||
* return MissionFitVerdictSchema.parse(raw);
|
||||
* One structured judge call. JUDGE_MODEL per the tiering rule in
|
||||
* models.ts — this verdict gates matches into/out of the review queue.
|
||||
* `MATCH_JUDGE_MODEL` env overrides for cheap bulk experiments.
|
||||
*/
|
||||
export async function runMissionFitJudge(_input: RunMissionFitJudgeInput): Promise<MissionFitVerdict> {
|
||||
throw new Error('not implemented');
|
||||
export async function runMatchJudge(
|
||||
input: RunMatchJudgeInput,
|
||||
): Promise<{ verdict: MatchJudgeVerdict; model: string }> {
|
||||
const model = process.env.MATCH_JUDGE_MODEL ?? JUDGE_MODEL;
|
||||
const result = await getAi().models.generateContent({
|
||||
model,
|
||||
contents: buildMatchJudgePrompt(input),
|
||||
config: {
|
||||
responseMimeType: 'application/json',
|
||||
responseJsonSchema: MATCH_JUDGE_VERDICT_JSON_SCHEMA,
|
||||
temperature: 0.1,
|
||||
},
|
||||
});
|
||||
const raw: unknown = JSON.parse(result.text ?? '{}');
|
||||
return { verdict: MatchJudgeVerdictSchema.parse(raw), model };
|
||||
}
|
||||
|
||||
@@ -1,20 +1,68 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Mission-fit judge verdict for one (org, grant) match candidate. This is
|
||||
* the last human-facing gate before a match is surfaced for review — the
|
||||
* judge must ground its verdict in something concrete from each side rather
|
||||
* than a vibe, so `citedOrgProgram` / `citedGrantPriority` are required, not
|
||||
* optional summary fields.
|
||||
* Graded mission-fit verdict for one (org, grant) match candidate that
|
||||
* already passed the deterministic hard gates and subscore ranking.
|
||||
*
|
||||
* The judge names a verdict; deterministic code assigns the number
|
||||
* (missionFitFromVerdict) and decides queue viability
|
||||
* (verdictIsFitViable) — the LLM never emits a score directly, so the
|
||||
* mapping can be re-tuned from review data without re-judging anything.
|
||||
*
|
||||
* Citations are required, not optional summary fields: a verdict must be
|
||||
* grounded in something concrete from each side rather than a vibe.
|
||||
*/
|
||||
export const MissionFitVerdictSchema = z.object({
|
||||
/** Whether the org's mission plausibly fits the grant's funding priorities. */
|
||||
fit: z.boolean(),
|
||||
/** The specific org program/activity the verdict is grounded in (from the org profile). */
|
||||
citedOrgProgram: z.string().min(1),
|
||||
/** The specific funding priority/eligibility line the verdict is grounded in (from the grant). */
|
||||
citedGrantPriority: z.string().min(1),
|
||||
/** Short human-readable justification tying the two citations together. */
|
||||
export const MatchJudgeVerdictSchema = z.object({
|
||||
verdict: z.enum(['strong_fit', 'plausible', 'weak', 'mismatch']),
|
||||
/** The specific org program/activity the verdict is grounded in. */
|
||||
citedOrgEvidence: z.string().min(1),
|
||||
/** The specific synopsis language (purpose/priorities) the verdict is grounded in. */
|
||||
citedGrantEvidence: z.string().min(1),
|
||||
/** 1–3 sentence justification tying the two citations together. */
|
||||
reasoning: z.string().min(1),
|
||||
});
|
||||
export type MissionFitVerdict = z.infer<typeof MissionFitVerdictSchema>;
|
||||
export type MatchJudgeVerdict = z.infer<typeof MatchJudgeVerdictSchema>;
|
||||
|
||||
/** Plain JSON Schema mirror of MatchJudgeVerdictSchema for `responseJsonSchema`. */
|
||||
export const MATCH_JUDGE_VERDICT_JSON_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
verdict: {
|
||||
type: 'string',
|
||||
enum: ['strong_fit', 'plausible', 'weak', 'mismatch'],
|
||||
},
|
||||
citedOrgEvidence: { type: 'string', minLength: 1 },
|
||||
citedGrantEvidence: { type: 'string', minLength: 1 },
|
||||
reasoning: { type: 'string', minLength: 1 },
|
||||
},
|
||||
required: ['verdict', 'citedOrgEvidence', 'citedGrantEvidence', 'reasoning'],
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Deterministic verdict → mission-fit subscore (0–30 scale, replacing the
|
||||
* embedding-band value on judged matches). `strong_fit` lands just under
|
||||
* the embedding maximum — a judge can rescue a good match the embeddings
|
||||
* missed, but only corroborated similarity reaches 30.
|
||||
*/
|
||||
export function missionFitFromVerdict(
|
||||
verdict: MatchJudgeVerdict['verdict'],
|
||||
): number {
|
||||
switch (verdict) {
|
||||
case 'strong_fit':
|
||||
return 27;
|
||||
case 'plausible':
|
||||
return 18;
|
||||
case 'weak':
|
||||
return 8;
|
||||
case 'mismatch':
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a judged match stays visible in the pending review queue. */
|
||||
export function verdictIsFitViable(
|
||||
verdict: MatchJudgeVerdict['verdict'],
|
||||
): boolean {
|
||||
return verdict === 'strong_fit' || verdict === 'plausible';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user