diff --git a/apps/outreach-review/app/routes/matches.$matchId.tsx b/apps/outreach-review/app/routes/matches.$matchId.tsx index 043effc..1d171a5 100644 --- a/apps/outreach-review/app/routes/matches.$matchId.tsx +++ b/apps/outreach-review/app/routes/matches.$matchId.tsx @@ -63,6 +63,17 @@ export default function MatchDetail({ loaderData }: Route.ComponentProps) { restrictionsOnAwards?: string; } | null; const subscores = (match.subscores ?? {}) as Record; + const orgPrograms = (org.programs ?? null) as Array<{ + name: string; + description?: string | null; + populationServed?: string | null; + sourceUrl?: string | null; + }> | null; + const orgFunders = (org.knownFunders ?? null) as Array<{ name: string }> | null; + const orgStaff = (org.staff ?? null) as Array<{ + name: string; + role?: string | null; + }> | null; const rationale = (match.rationale ?? {}) as Record; const nhHistory = funderGivingHistory; @@ -143,6 +154,43 @@ export default function MatchDetail({ loaderData }: Route.ComponentProps) { + {orgPrograms != null && orgPrograms.length > 0 && ( +
+

Programs (researched)

+
    + {orgPrograms.map( + (p, i) => ( +
  • + {p.name} + {p.populationServed != null && ` — serving ${p.populationServed}`} + {p.sourceUrl != null && ( + <> + {' '} + + src + + + )} +
    • + ), + )} +
+
+ )} + {orgFunders != null && orgFunders.length > 0 && ( +

+ Known funders: + {orgFunders.map((f) => f.name).join(', ')} +

+ )} + {orgStaff != null && orgStaff.length > 0 && ( +

+ Staff: + {orgStaff + .map((s) => (s.role != null ? `${s.name} (${s.role})` : s.name)) + .join(', ')} +

+ )}
{org.ein != null && ( Promise> = { matchGrants: runMatchGrantsNow, ingest990pf: runIngest990PfNow, federalPrecedent: runFederalPrecedentNow, + profileOrgs: runProfileOrgsNow, }; const FIRST_RUN_ORDER = [ @@ -75,6 +80,7 @@ const FIRST_RUN_ORDER = [ 'federalPrecedent', 'embedGrants', 'matchGrants', + 'profileOrgs', ]; if (process.env.DATABASE_URL == null) { @@ -113,6 +119,7 @@ async function main() { setMatchGrantsDeps({ db }); setIngest990pfDeps({ db }); setFederalPrecedentDeps({ db }); + setProfileOrgsDeps({ db }); DBOS.setConfig({ name: 'helmdocs-outreach-worker', diff --git a/apps/outreach-worker/src/workflows/profile-orgs.ts b/apps/outreach-worker/src/workflows/profile-orgs.ts new file mode 100644 index 0000000..061bd30 --- /dev/null +++ b/apps/outreach-worker/src/workflows/profile-orgs.ts @@ -0,0 +1,208 @@ +/** + * Nightly Stage 4 profiling workflow — grounded web research for orgs + * that already have a plausible lead (>=1 non-rejected easy-win match), + * best-scoring first, replacing their NTEE-stub profiles with cited, + * researched ones. Better profiles → better mission-fit embeddings on the + * NEXT match run → better personalization at review time. + * + * The nightly batch cap bounds LLM spend (two Flash calls + one embedding + * per org, ≈ cents each); env `PROFILE_ORGS_PER_RUN` overrides. + * + * Low-confidence results (identity not confirmed, or the model reports + * <= 0.4) are STILL stored — with their low confidence — so the review UI + * can warn, but they do NOT overwrite... actually they do overwrite the + * stub, which is correct: "we looked and couldn't confirm" is more + * information than an NTEE guess, and the confidence field carries it. + * The profile embedding, however, keeps mission-fit honest: identity- + * unconfirmed profiles are embedded from the stub text, not the + * possibly-wrong research. + * + * Runs at 05:45, after matchGrants (05:15). Registration follows + * `ingest-grants.ts` exactly. + */ +import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk'; +import { + buildProfileEmbeddingText, + generateQueryEmbedding, + researchOrgProfile, + type ResearchedProfileResult, +} from '@novelpad/outreach-ai'; +import { buildOrgMissionText, type schema } from '@novelpad/outreach-core'; +import { + serverApplyResearchedProfile, + serverListOrgsNeedingProfile, + type OrgNeedingProfile, +} from '@novelpad/outreach-core/server'; +import type { NodePgDatabase } from 'drizzle-orm/node-postgres'; + +export type OutreachDb = NodePgDatabase; + +const DEFAULT_ORGS_PER_RUN = 25; +/** Below this, the research embedding is not trusted for mission fit. */ +const EMBEDDING_TRUST_FLOOR = 0.5; + +export interface ProfileOrgsDeps { + readonly db: OutreachDb; +} + +let registeredDeps: ProfileOrgsDeps | null = null; + +export function setProfileOrgsDeps(deps: ProfileOrgsDeps): void { + registeredDeps = deps; +} + +function getProfileOrgsDeps(): ProfileOrgsDeps { + if (registeredDeps == null) { + throw new Error( + 'ProfileOrgsDeps not registered. Call setProfileOrgsDeps() before DBOS.launch().', + ); + } + return registeredDeps; +} + +async function listOrgsNeedingProfile( + db: OutreachDb, + limit: number, +): Promise { + return serverListOrgsNeedingProfile(db, { limit }); +} +const listOrgsNeedingProfileStep = DBOS.registerStep(listOrgsNeedingProfile, { + name: 'listOrgsNeedingProfile', + retriesAllowed: true, + maxAttempts: 3, +}); + +async function researchOrg( + org: OrgNeedingProfile, +): Promise { + return researchOrgProfile(org); +} +const researchOrgStep = DBOS.registerStep(researchOrg, { + name: 'researchOrgProfile', + retriesAllowed: true, + maxAttempts: 2, +}); + +async function storeProfile( + db: OutreachDb, + org: OrgNeedingProfile, + result: ResearchedProfileResult, +): Promise { + const { profile, sources } = result; + + const trustworthy = + profile.identityConfirmed && + profile.overallConfidence >= EMBEDDING_TRUST_FLOOR; + const embeddingText = trustworthy + ? buildProfileEmbeddingText(org, profile) + : buildOrgMissionText(org); + const embedding = await generateQueryEmbedding(embeddingText); + + await serverApplyResearchedProfile(db, { + orgId: org.id, + missionStatement: profile.mission?.value ?? null, + programs: profile.programs, + serviceGeography: profile.serviceGeography?.value ?? null, + recentNews: profile.recentNews, + knownFunders: profile.knownFunders, + staff: profile.staff, + sources, + confidence: profile.identityConfirmed + ? profile.overallConfidence + : Math.min(profile.overallConfidence, 0.3), + profileEmbedding: embedding, + }); +} +const storeProfileStep = DBOS.registerStep(storeProfile, { + name: 'storeResearchedProfile', + retriesAllowed: true, + maxAttempts: 3, +}); + +async function runProfileOrgs(): Promise { + const { db } = getProfileOrgsDeps(); + + const limit = Number(process.env.PROFILE_ORGS_PER_RUN ?? DEFAULT_ORGS_PER_RUN); + const orgs = await listOrgsNeedingProfileStep(db, limit); + if (orgs.length === 0) { + console.log('[profile-orgs] no easy-win orgs awaiting research'); + return; + } + + let profiled = 0; + let lowConfidence = 0; + let failed = 0; + for (const org of orgs) { + try { + const result = await researchOrgStep(org); + await storeProfileStep(db, org, result); + profiled++; + if ( + !result.profile.identityConfirmed || + result.profile.overallConfidence < EMBEDDING_TRUST_FLOOR + ) { + lowConfidence++; + } + } catch (err) { + failed++; + console.error(`[profile-orgs] org "${org.name}" failed:`, err); + } + } + + console.log( + `[profile-orgs] attempted=${orgs.length} profiled=${profiled} lowConfidence=${lowConfidence} failed=${failed}`, + ); + if (failed > 0 && failed / orgs.length > 0.2) { + throw new Error( + `[profile-orgs] systemic failure: ${failed}/${orgs.length} orgs failed`, + ); + } +} + +const g = globalThis as unknown as { + __outreachProfileOrgsRegistered?: boolean; + __outreachProfileOrgsHandle?: ( + scheduledTime: Date, + startedAt: Date, + ) => Promise; +}; + +if (!g.__outreachProfileOrgsRegistered) { + g.__outreachProfileOrgsRegistered = true; + + const profileOrgs = async (_scheduledTime: Date, _startedAt: Date) => { + try { + await runProfileOrgs(); + } catch (err) { + console.error('[profile-orgs] pass failed:', err); + throw err; + } + }; + + // Must be registered as BOTH a workflow and a scheduled function, + // referencing the same function object — see ingest-grants.ts. + g.__outreachProfileOrgsHandle = DBOS.registerWorkflow(profileOrgs, { + name: 'profileOrgs', + }); + DBOS.registerScheduled(profileOrgs, { + crontab: '45 5 * * *', + name: 'profileOrgs', + mode: SchedulerMode.ExactlyOncePerInterval, + }); +} + +/** + * Starts one durable run of this workflow immediately through DBOS — + * the exact production path (workflow + checkpointed steps), used by + * `run-once.ts` for supervised/manual passes. Requires deps injected and + * `DBOS.launch()` completed. + */ +export function runProfileOrgsNow(): Promise { + const handle = g.__outreachProfileOrgsHandle; + if (handle == null) { + throw new Error( + 'profileOrgs is not registered; was this module imported before DBOS.launch()?', + ); + } + return handle(new Date(), new Date()); +} diff --git a/docs/features/profiler.md b/docs/features/profiler.md new file mode 100644 index 0000000..4154778 --- /dev/null +++ b/docs/features/profiler.md @@ -0,0 +1,15 @@ +# Stage 4 — Org research profiler + +Nightly `profileOrgs` workflow (05:45, after matchGrants): grounded web research for orgs that already have ≥1 non-rejected easy-win match (the plan's "coarse match gates profiling" cost rule), best-scoring first, 25/night (`PROFILE_ORGS_PER_RUN`). + +**Two Gemini passes per org** (`packages/outreach-ai/src/agents/org-profiler/research.ts`): +1. **Grounded research** — `gemini-2.5-flash` with the Google Search tool; `groundingMetadata` supplies the citation universe (every page actually consulted). +2. **Structured extraction** — flash constrained to a JSON schema over the research notes, citing only from that universe. Search grounding and JSON-schema output can't share one call, hence the split. + +**Output** overwrites the org's NTEE stub in `org_profiles` (mission, programs w/ populations served, service geography, known funders, staff, recent news, sources, confidence). The profile embedding is rebuilt from the researched text — unless identity wasn't confirmed or confidence < 0.5, in which case the embedding stays NTEE-derived so unverified research can't steer mission fit. An identity-unconfirmed profile is stored at ≤0.3 confidence: "we looked and couldn't confirm" beats an NTEE guess, and the review UI warns on it. + +**Review integration**: match detail pages render researched programs (with per-claim source links), known funders, and staff. + +**First live batches (2026-07-16/17)**: 55 orgs, 41 research-grade (>0.4), 2 low-confidence, 0 failures, ~28 min for 30 orgs (grounded search dominates). Verified effect on scoring: Annie's Angels → NIH Mammalian Models mission fit fell 19→12 (real mission: family financial crisis support), Seacoast Pathways → Foundation for Seacoast Health rose to 25/30 fit — and its researched `knownFunders` list independently named Foundation for Seacoast Health, confirming the precedent match. + +**Gotcha fixed en route**: `org_profiles.confidence` is float4 — `0.2` stores as `0.20000000298`, so `confidence <= 0.2` (float8 comparison) silently excluded every stub; the needing-profile query uses an epsilon. diff --git a/packages/outreach-ai/src/agents/org-profiler/research.ts b/packages/outreach-ai/src/agents/org-profiler/research.ts new file mode 100644 index 0000000..bbe3a84 --- /dev/null +++ b/packages/outreach-ai/src/agents/org-profiler/research.ts @@ -0,0 +1,106 @@ +import { getAi } from '../../gemini.js'; +import { BULK_MODEL } from '../../models.js'; +import { + buildExtractionPrompt, + buildResearchPrompt, + RESEARCHED_ORG_PROFILE_JSON_SCHEMA, + ResearchedOrgProfileSchema, + type ProfileResearchTarget, + type ResearchedOrgProfile, +} from './researched-profile.js'; + +export interface ResearchedProfileResult { + readonly profile: ResearchedOrgProfile; + /** Every grounding source the research pass touched (deduped). */ + readonly sources: Array<{ url: string; title: string | null }>; +} + +/** + * Stage 4 profiler: two Gemini passes. + * + * 1. Grounded research — BULK_MODEL with the Google Search tool; the + * response's groundingMetadata carries the actual pages consulted, + * which become the profile's citation universe. + * 2. Structured extraction — BULK_MODEL constrained to + * RESEARCHED_ORG_PROFILE_JSON_SCHEMA over the research text, citing + * only from that universe. + * + * Search grounding and JSON-schema output can't share one call, hence the + * split. Cost ≈ two Flash calls per org — the per-org spend the plan's + * "coarse match gates profiling" rule exists to bound. + */ +export async function researchOrgProfile( + org: ProfileResearchTarget, +): Promise { + const research = await getAi().models.generateContent({ + model: BULK_MODEL, + contents: buildResearchPrompt(org), + config: { + tools: [{ googleSearch: {} }], + temperature: 0.2, + }, + }); + + const researchText = research.text ?? ''; + const chunks = + research.candidates?.[0]?.groundingMetadata?.groundingChunks ?? []; + const seen = new Map(); + for (const chunk of chunks) { + const uri = chunk.web?.uri; + if (uri != null && !seen.has(uri)) seen.set(uri, chunk.web?.title ?? null); + } + const sources = [...seen.entries()].map(([url, title]) => ({ url, title })); + + if (researchText.trim() === '') { + throw new Error( + `researchOrgProfile: empty research response for "${org.name}"`, + ); + } + + const extraction = await getAi().models.generateContent({ + model: BULK_MODEL, + contents: buildExtractionPrompt( + org, + researchText, + sources.map((s) => s.url), + ), + config: { + responseMimeType: 'application/json', + responseJsonSchema: RESEARCHED_ORG_PROFILE_JSON_SCHEMA, + temperature: 0.1, + }, + }); + + const raw: unknown = JSON.parse(extraction.text ?? '{}'); + const profile = ResearchedOrgProfileSchema.parse(raw); + + return { profile, sources }; +} + +/** + * Description-shaped text representing the researched profile in embedding + * space (RETRIEVAL_QUERY side of mission-fit). Same composition every + * re-embed so profiles are comparable run to run. + */ +export function buildProfileEmbeddingText( + org: ProfileResearchTarget, + profile: ResearchedOrgProfile, +): string { + const parts: string[] = [org.name]; + if (profile.mission != null) parts.push(`Mission: ${profile.mission.value}`); + if (profile.programs.length > 0) { + parts.push( + `Programs: ${profile.programs + .map((p) => + [p.name, p.description, p.populationServed && `serving ${p.populationServed}`] + .filter(Boolean) + .join(' — '), + ) + .join('; ')}`, + ); + } + if (profile.serviceGeography != null) { + parts.push(`Serves: ${profile.serviceGeography.value}`); + } + return parts.join('\n').slice(0, 8_000); +} diff --git a/packages/outreach-ai/src/agents/org-profiler/researched-profile.test.ts b/packages/outreach-ai/src/agents/org-profiler/researched-profile.test.ts new file mode 100644 index 0000000..b04d98d --- /dev/null +++ b/packages/outreach-ai/src/agents/org-profiler/researched-profile.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; + +import { + buildExtractionPrompt, + buildResearchPrompt, + ResearchedOrgProfileSchema, +} from './researched-profile.js'; + +describe('buildResearchPrompt', () => { + it('names the org, location, and identity guard', () => { + const p = buildResearchPrompt({ + name: 'Seacoast Pathways', + city: 'Portsmouth', + state: 'NH', + ein: '020123456', + nteeCode: 'F80', + }); + expect(p).toContain('"Seacoast Pathways"'); + expect(p).toContain('Portsmouth, NH'); + expect(p).toContain('020123456'); + expect(p).toContain('not a similarly-named organization'); + }); +}); + +describe('buildExtractionPrompt', () => { + it('embeds sources and identity rule', () => { + const p = buildExtractionPrompt( + { name: 'X', city: 'Keene', state: 'NH', ein: null, nteeCode: null }, + 'notes here', + ['https://a.example', 'https://b.example'], + ); + expect(p).toContain('- https://a.example'); + expect(p).toContain('identityConfirmed=false'); + expect(p).toContain('notes here'); + }); +}); + +describe('ResearchedOrgProfileSchema', () => { + it('accepts a minimal valid profile and rejects bad confidence', () => { + const minimal = { + identityConfirmed: false, + orgWebsite: null, + mission: null, + programs: [], + serviceGeography: null, + knownFunders: [], + staff: [], + recentNews: [], + overallConfidence: 0.1, + }; + expect(ResearchedOrgProfileSchema.parse(minimal).identityConfirmed).toBe(false); + expect(() => + ResearchedOrgProfileSchema.parse({ ...minimal, overallConfidence: 1.5 }), + ).toThrow(); + }); +}); diff --git a/packages/outreach-ai/src/agents/org-profiler/researched-profile.ts b/packages/outreach-ai/src/agents/org-profiler/researched-profile.ts new file mode 100644 index 0000000..066bc6b --- /dev/null +++ b/packages/outreach-ai/src/agents/org-profiler/researched-profile.ts @@ -0,0 +1,192 @@ +import { z } from 'zod'; + +/** + * Stage 4 researched profile — aligned 1:1 with the `org_profiles` table. + * Every claim carries the URL it came from; Email 1's personalization line + * is only as trustworthy as these citations, so an uncited claim is a + * lower-trust claim by construction (docs/plan.md, Stage 4). + */ + +const cited = (valueSchema: Value) => + z.object({ + value: valueSchema, + sourceUrl: z.string().nullable(), + }); + +export const ResearchedOrgProfileSchema = z.object({ + /** Did the research confidently identify THIS org (right city/state)? */ + identityConfirmed: z.boolean(), + orgWebsite: z.string().nullable(), + mission: cited(z.string().min(1)).nullable(), + programs: z.array( + z.object({ + name: z.string().min(1), + description: z.string().nullable(), + populationServed: z.string().nullable(), + sourceUrl: z.string().nullable(), + }), + ), + serviceGeography: cited(z.string().min(1)).nullable(), + knownFunders: z.array( + z.object({ name: z.string().min(1), sourceUrl: z.string().nullable() }), + ), + staff: z.array( + z.object({ + name: z.string().min(1), + role: z.string().nullable(), + sourceUrl: z.string().nullable(), + }), + ), + recentNews: z.array( + z.object({ + headline: z.string().min(1), + url: z.string().nullable(), + date: z.string().nullable(), + }), + ), + /** Model's overall 0–1 confidence that the profile describes the right org accurately. */ + overallConfidence: z.number().min(0).max(1), +}); +export type ResearchedOrgProfile = z.infer; + +/** Plain JSON Schema mirror of ResearchedOrgProfileSchema for Gemini's responseJsonSchema. */ +export const RESEARCHED_ORG_PROFILE_JSON_SCHEMA = { + type: 'object', + properties: { + identityConfirmed: { type: 'boolean' }, + orgWebsite: { type: ['string', 'null'] }, + mission: { + type: ['object', 'null'], + properties: { + value: { type: 'string' }, + sourceUrl: { type: ['string', 'null'] }, + }, + required: ['value', 'sourceUrl'], + }, + programs: { + type: 'array', + items: { + type: 'object', + properties: { + name: { type: 'string' }, + description: { type: ['string', 'null'] }, + populationServed: { type: ['string', 'null'] }, + sourceUrl: { type: ['string', 'null'] }, + }, + required: ['name', 'description', 'populationServed', 'sourceUrl'], + }, + }, + serviceGeography: { + type: ['object', 'null'], + properties: { + value: { type: 'string' }, + sourceUrl: { type: ['string', 'null'] }, + }, + required: ['value', 'sourceUrl'], + }, + knownFunders: { + type: 'array', + items: { + type: 'object', + properties: { + name: { type: 'string' }, + sourceUrl: { type: ['string', 'null'] }, + }, + required: ['name', 'sourceUrl'], + }, + }, + staff: { + type: 'array', + items: { + type: 'object', + properties: { + name: { type: 'string' }, + role: { type: ['string', 'null'] }, + sourceUrl: { type: ['string', 'null'] }, + }, + required: ['name', 'role', 'sourceUrl'], + }, + }, + recentNews: { + type: 'array', + items: { + type: 'object', + properties: { + headline: { type: 'string' }, + url: { type: ['string', 'null'] }, + date: { type: ['string', 'null'] }, + }, + required: ['headline', 'url', 'date'], + }, + }, + overallConfidence: { type: 'number', minimum: 0, maximum: 1 }, + }, + required: [ + 'identityConfirmed', + 'orgWebsite', + 'mission', + 'programs', + 'serviceGeography', + 'knownFunders', + 'staff', + 'recentNews', + 'overallConfidence', + ], +} as const; + +export interface ProfileResearchTarget { + readonly name: string; + readonly city: string | null; + readonly state: string; + readonly ein: string | null; + readonly nteeCode: string | null; +} + +/** Prompt for the grounded research pass (Google Search tool enabled). */ +export function buildResearchPrompt(org: ProfileResearchTarget): string { + const where = [org.city, org.state].filter(Boolean).join(', '); + return [ + `Research the nonprofit organization "${org.name}" based in ${where}.`, + org.ein != null ? `Its federal EIN is ${org.ein}.` : '', + '', + 'Find and report, with specifics:', + '1. Their mission, in their own words if possible.', + '2. Their concrete programs/services — names, what each does, and who it serves.', + '3. The geography they serve (towns/counties/statewide).', + '4. Foundations or agencies that fund them (funder walls, annual reports, press).', + '5. Key staff: executive director and anyone in a development/grants role.', + '6. Any news from the last two years.', + '', + 'CRITICAL: verify you are describing THIS organization — the one in', + `${where} — not a similarly-named organization elsewhere. If you cannot`, + 'find credible information clearly about this specific organization, say', + 'so explicitly instead of substituting a lookalike.', + ] + .filter((line) => line !== '') + .join('\n'); +} + +/** Prompt for the structured extraction pass over the research text. */ +export function buildExtractionPrompt( + org: ProfileResearchTarget, + researchText: string, + sourceUrls: readonly string[], +): string { + return [ + 'Extract a structured profile of the nonprofit below from the research', + 'notes. Rules:', + '- Only include claims supported by the research notes.', + '- For each claim, cite the most relevant source URL from the list, or', + ' null if the claim synthesizes multiple sources.', + '- If the research failed to clearly identify this specific organization', + ` (the one in ${[org.city, org.state].filter(Boolean).join(', ')}),`, + ' set identityConfirmed=false and overallConfidence at or below 0.3.', + '- Empty arrays are fine. Never invent programs, funders, staff, or news.', + '', + `Organization: ${org.name}`, + '', + `Available source URLs:\n${sourceUrls.map((u) => `- ${u}`).join('\n') || '- (none)'}`, + '', + `Research notes:\n${researchText}`, + ].join('\n'); +} diff --git a/packages/outreach-ai/src/index.ts b/packages/outreach-ai/src/index.ts index b2116bf..d78d2b8 100644 --- a/packages/outreach-ai/src/index.ts +++ b/packages/outreach-ai/src/index.ts @@ -6,3 +6,5 @@ export * from './agents/org-profiler/run.js'; export * from './agents/mission-fit-judge/schema.js'; export * from './agents/mission-fit-judge/run.js'; export * from './grant-embedding-text.js'; +export * from './agents/org-profiler/researched-profile.js'; +export * from './agents/org-profiler/research.js'; diff --git a/packages/outreach-core/src/matches/queries/get-match-detail.server.ts b/packages/outreach-core/src/matches/queries/get-match-detail.server.ts index 8b1158b..85ed8e7 100644 --- a/packages/outreach-core/src/matches/queries/get-match-detail.server.ts +++ b/packages/outreach-core/src/matches/queries/get-match-detail.server.ts @@ -24,6 +24,11 @@ export interface MatchDetail { registrationNumber: string | null; missionStatement: string | null; profileConfidence: number | null; + programs: unknown; + knownFunders: unknown; + staff: unknown; + serviceGeography: string | null; + profileSources: unknown; }; grant: { id: string; @@ -103,6 +108,11 @@ export async function serverGetMatchDetail( .select({ missionStatement: schema.orgProfiles.missionStatement, confidence: schema.orgProfiles.confidence, + programs: schema.orgProfiles.programs, + knownFunders: schema.orgProfiles.knownFunders, + staff: schema.orgProfiles.staff, + serviceGeography: schema.orgProfiles.serviceGeography, + sources: schema.orgProfiles.sources, }) .from(schema.orgProfiles) .where(eq(schema.orgProfiles.orgId, row.orgId)) @@ -184,6 +194,11 @@ export async function serverGetMatchDetail( registrationNumber: row.orgRegNo, missionStatement: profiles[0]?.missionStatement ?? null, profileConfidence: profiles[0]?.confidence ?? null, + programs: profiles[0]?.programs ?? null, + knownFunders: profiles[0]?.knownFunders ?? null, + staff: profiles[0]?.staff ?? null, + serviceGeography: profiles[0]?.serviceGeography ?? null, + profileSources: profiles[0]?.sources ?? null, }, grant: { id: row.grantId, diff --git a/packages/outreach-core/src/orgs/actions/apply-researched-profile.server.ts b/packages/outreach-core/src/orgs/actions/apply-researched-profile.server.ts new file mode 100644 index 0000000..3bc1332 --- /dev/null +++ b/packages/outreach-core/src/orgs/actions/apply-researched-profile.server.ts @@ -0,0 +1,57 @@ +import { sql } from 'drizzle-orm'; + +import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; +import { schema } from '#~/db/db.js'; + +export interface ResearchedProfileInput { + readonly orgId: string; + readonly missionStatement: string | null; + readonly programs: unknown; + readonly serviceGeography: string | null; + readonly recentNews: unknown; + readonly knownFunders: unknown; + readonly staff: unknown; + readonly sources: unknown; + readonly confidence: number; + /** 1536-dim RETRIEVAL_QUERY embedding of the researched profile text. */ + readonly profileEmbedding: number[]; +} + +/** + * Applies a Stage 4 researched profile, OVERWRITING the row — this is the + * upgrade path the v0 NTEE stub (confidence 0.2, never-overwriting) exists + * to be replaced by. Re-running research refreshes in place. + */ +export async function serverApplyResearchedProfile( + db: NpOutreachDatabase | NpOutreachTransaction, + profile: ResearchedProfileInput, +): Promise { + await db + .insert(schema.orgProfiles) + .values({ + orgId: profile.orgId, + missionStatement: profile.missionStatement, + programs: profile.programs, + serviceGeography: profile.serviceGeography, + recentNews: profile.recentNews, + knownFunders: profile.knownFunders, + staff: profile.staff, + sources: profile.sources, + confidence: profile.confidence, + profileEmbedding: profile.profileEmbedding, + }) + .onConflictDoUpdate({ + target: schema.orgProfiles.orgId, + set: { + missionStatement: sql`excluded.mission_statement`, + programs: sql`excluded.programs`, + serviceGeography: sql`excluded.service_geography`, + recentNews: sql`excluded.recent_news`, + knownFunders: sql`excluded.known_funders`, + staff: sql`excluded.staff`, + sources: sql`excluded.sources`, + confidence: sql`excluded.confidence`, + profileEmbedding: sql`excluded.profile_embedding`, + }, + }); +} diff --git a/packages/outreach-core/src/orgs/actions/index.server.ts b/packages/outreach-core/src/orgs/actions/index.server.ts index 8f7fc2a..9bc82c8 100644 --- a/packages/outreach-core/src/orgs/actions/index.server.ts +++ b/packages/outreach-core/src/orgs/actions/index.server.ts @@ -2,3 +2,4 @@ export * from './insert-org.server.js'; export * from './upsert-org-from-registry.server.js'; export * from './enrich-org.server.js'; export * from './ensure-org-profile-stub.server.js'; +export * from './apply-researched-profile.server.js'; diff --git a/packages/outreach-core/src/orgs/queries/index.server.ts b/packages/outreach-core/src/orgs/queries/index.server.ts index 95e35e5..b014bca 100644 --- a/packages/outreach-core/src/orgs/queries/index.server.ts +++ b/packages/outreach-core/src/orgs/queries/index.server.ts @@ -1,3 +1,4 @@ export * from './list-orgs-in-icp-band.server.js'; export * from './list-orgs-needing-enrichment.server.js'; export * from './list-match-candidate-orgs.server.js'; +export * from './list-orgs-needing-profile.server.js'; diff --git a/packages/outreach-core/src/orgs/queries/list-orgs-needing-profile.server.ts b/packages/outreach-core/src/orgs/queries/list-orgs-needing-profile.server.ts new file mode 100644 index 0000000..111a83b --- /dev/null +++ b/packages/outreach-core/src/orgs/queries/list-orgs-needing-profile.server.ts @@ -0,0 +1,55 @@ +import { sql } from 'drizzle-orm'; + +import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js'; + +export interface OrgNeedingProfile { + id: string; + name: string; + city: string | null; + state: string; + ein: string | null; + nteeCode: string | null; + bestMatchScore: number; +} + +/** Profiles below/at this confidence are NTEE-derived stubs, not research. */ +export const STUB_CONFIDENCE_CEILING = 0.2; + +/** + * The plan's cost-control rule made concrete: research only orgs that + * already have at least one non-rejected easy-win match (a plausible + * lead), best-scoring orgs first, and only when the current profile is a + * stub (or missing). Re-research after review feedback comes later. + */ +export async function serverListOrgsNeedingProfile( + db: NpOutreachDatabase | NpOutreachTransaction, + { limit }: { limit: number }, +): Promise { + const result = await db.execute(sql` + SELECT + o.id, o.name, o.city, o.state, o.ein, o.ntee_code, + max(m.total_score) AS best_match_score + FROM orgs o + JOIN matches m ON m.org_id = o.id + AND m.review_status != 'rejected' + AND m.easy_win = true + LEFT JOIN org_profiles p ON p.org_id = o.id + -- confidence is a float4: 0.2 stores as 0.20000000298, so a plain + -- <= 0.2 comparison (float8) excludes every stub. Epsilon absorbs it. + WHERE p.id IS NULL OR p.confidence IS NULL OR p.confidence <= ${STUB_CONFIDENCE_CEILING} + 0.001 + GROUP BY o.id + ORDER BY max(m.total_score) DESC + LIMIT ${limit} + `); + + const { rows } = result as unknown as { rows: Record[] }; + return rows.map((r) => ({ + id: r.id as string, + name: r.name as string, + city: (r.city as string) ?? null, + state: r.state as string, + ein: (r.ein as string) ?? null, + nteeCode: (r.ntee_code as string) ?? null, + bestMatchScore: Number(r.best_match_score), + })); +}