feat(profiler): Stage 4 grounded research profiler — cited profiles replace NTEE stubs

profileOrgs (nightly 05:45, 25/run, easy-win orgs best-first per the
plan's coarse-match-gates-profiling rule): two-pass Gemini flash —
Google Search-grounded research (groundingMetadata = citation universe)
then JSON-schema extraction citing only from it. Overwrites the stub
profile (mission/programs/geography/funders/staff/news/sources/
confidence); re-embeds mission fit from researched text unless identity
unconfirmed or confidence <0.5 (then the NTEE embedding stays —
unverified research must not steer scoring). Detail page renders
researched programs w/ source links, funders, staff.

Fixes: org_profiles.confidence is float4 — 0.2 stores as 0.20000000298
so 'confidence <= 0.2' excluded every stub (epsilon comparison); RR7
loader serialization turns unknown into never (hoisted casts).

Live: 55 orgs profiled (41 research-grade, 2 low-confidence, 0 failed).
Scoring effect verified both directions: Annie's Angels x NIH mammalian
models fit 19->12; Seacoast Pathways x Foundation for Seacoast Health
fit 25/30 — and the researched knownFunders independently names that
funder. 161 tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

packages/outreach-ai/src/agents/org-profiler/research.ts

@@ -0,0 +1,106 @@
+import { getAi } from '../../gemini.js';
+import { BULK_MODEL } from '../../models.js';
+import {
+  buildExtractionPrompt,
+  buildResearchPrompt,
+  RESEARCHED_ORG_PROFILE_JSON_SCHEMA,
+  ResearchedOrgProfileSchema,
+  type ProfileResearchTarget,
+  type ResearchedOrgProfile,
+} from './researched-profile.js';
+
+export interface ResearchedProfileResult {
+  readonly profile: ResearchedOrgProfile;
+  /** Every grounding source the research pass touched (deduped). */
+  readonly sources: Array<{ url: string; title: string | null }>;
+}
+
+/**
+ * Stage 4 profiler: two Gemini passes.
+ *
+ *  1. Grounded research — BULK_MODEL with the Google Search tool; the
+ *     response's groundingMetadata carries the actual pages consulted,
+ *     which become the profile's citation universe.
+ *  2. Structured extraction — BULK_MODEL constrained to
+ *     RESEARCHED_ORG_PROFILE_JSON_SCHEMA over the research text, citing
+ *     only from that universe.
+ *
+ * Search grounding and JSON-schema output can't share one call, hence the
+ * split. Cost ≈ two Flash calls per org — the per-org spend the plan's
+ * "coarse match gates profiling" rule exists to bound.
+ */
+export async function researchOrgProfile(
+  org: ProfileResearchTarget,
+): Promise<ResearchedProfileResult> {
+  const research = await getAi().models.generateContent({
+    model: BULK_MODEL,
+    contents: buildResearchPrompt(org),
+    config: {
+      tools: [{ googleSearch: {} }],
+      temperature: 0.2,
+    },
+  });
+
+  const researchText = research.text ?? '';
+  const chunks =
+    research.candidates?.[0]?.groundingMetadata?.groundingChunks ?? [];
+  const seen = new Map<string, string | null>();
+  for (const chunk of chunks) {
+    const uri = chunk.web?.uri;
+    if (uri != null && !seen.has(uri)) seen.set(uri, chunk.web?.title ?? null);
+  }
+  const sources = [...seen.entries()].map(([url, title]) => ({ url, title }));
+
+  if (researchText.trim() === '') {
+    throw new Error(
+      `researchOrgProfile: empty research response for "${org.name}"`,
+    );
+  }
+
+  const extraction = await getAi().models.generateContent({
+    model: BULK_MODEL,
+    contents: buildExtractionPrompt(
+      org,
+      researchText,
+      sources.map((s) => s.url),
+    ),
+    config: {
+      responseMimeType: 'application/json',
+      responseJsonSchema: RESEARCHED_ORG_PROFILE_JSON_SCHEMA,
+      temperature: 0.1,
+    },
+  });
+
+  const raw: unknown = JSON.parse(extraction.text ?? '{}');
+  const profile = ResearchedOrgProfileSchema.parse(raw);
+
+  return { profile, sources };
+}
+
+/**
+ * Description-shaped text representing the researched profile in embedding
+ * space (RETRIEVAL_QUERY side of mission-fit). Same composition every
+ * re-embed so profiles are comparable run to run.
+ */
+export function buildProfileEmbeddingText(
+  org: ProfileResearchTarget,
+  profile: ResearchedOrgProfile,
+): string {
+  const parts: string[] = [org.name];
+  if (profile.mission != null) parts.push(`Mission: ${profile.mission.value}`);
+  if (profile.programs.length > 0) {
+    parts.push(
+      `Programs: ${profile.programs
+        .map((p) =>
+          [p.name, p.description, p.populationServed && `serving ${p.populationServed}`]
+            .filter(Boolean)
+            .join(' — '),
+        )
+        .join('; ')}`,
+    );
+  }
+  if (profile.serviceGeography != null) {
+    parts.push(`Serves: ${profile.serviceGeography.value}`);
+  }
+  return parts.join('\n').slice(0, 8_000);
+}

packages/outreach-ai/src/agents/org-profiler/researched-profile.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  buildExtractionPrompt,
+  buildResearchPrompt,
+  ResearchedOrgProfileSchema,
+} from './researched-profile.js';
+
+describe('buildResearchPrompt', () => {
+  it('names the org, location, and identity guard', () => {
+    const p = buildResearchPrompt({
+      name: 'Seacoast Pathways',
+      city: 'Portsmouth',
+      state: 'NH',
+      ein: '020123456',
+      nteeCode: 'F80',
+    });
+    expect(p).toContain('"Seacoast Pathways"');
+    expect(p).toContain('Portsmouth, NH');
+    expect(p).toContain('020123456');
+    expect(p).toContain('not a similarly-named organization');
+  });
+});
+
+describe('buildExtractionPrompt', () => {
+  it('embeds sources and identity rule', () => {
+    const p = buildExtractionPrompt(
+      { name: 'X', city: 'Keene', state: 'NH', ein: null, nteeCode: null },
+      'notes here',
+      ['https://a.example', 'https://b.example'],
+    );
+    expect(p).toContain('- https://a.example');
+    expect(p).toContain('identityConfirmed=false');
+    expect(p).toContain('notes here');
+  });
+});
+
+describe('ResearchedOrgProfileSchema', () => {
+  it('accepts a minimal valid profile and rejects bad confidence', () => {
+    const minimal = {
+      identityConfirmed: false,
+      orgWebsite: null,
+      mission: null,
+      programs: [],
+      serviceGeography: null,
+      knownFunders: [],
+      staff: [],
+      recentNews: [],
+      overallConfidence: 0.1,
+    };
+    expect(ResearchedOrgProfileSchema.parse(minimal).identityConfirmed).toBe(false);
+    expect(() =>
+      ResearchedOrgProfileSchema.parse({ ...minimal, overallConfidence: 1.5 }),
+    ).toThrow();
+  });
+});

packages/outreach-ai/src/agents/org-profiler/researched-profile.ts

@@ -0,0 +1,192 @@
+import { z } from 'zod';
+
+/**
+ * Stage 4 researched profile — aligned 1:1 with the `org_profiles` table.
+ * Every claim carries the URL it came from; Email 1's personalization line
+ * is only as trustworthy as these citations, so an uncited claim is a
+ * lower-trust claim by construction (docs/plan.md, Stage 4).
+ */
+
+const cited = <Value extends z.ZodTypeAny>(valueSchema: Value) =>
+  z.object({
+    value: valueSchema,
+    sourceUrl: z.string().nullable(),
+  });
+
+export const ResearchedOrgProfileSchema = z.object({
+  /** Did the research confidently identify THIS org (right city/state)? */
+  identityConfirmed: z.boolean(),
+  orgWebsite: z.string().nullable(),
+  mission: cited(z.string().min(1)).nullable(),
+  programs: z.array(
+    z.object({
+      name: z.string().min(1),
+      description: z.string().nullable(),
+      populationServed: z.string().nullable(),
+      sourceUrl: z.string().nullable(),
+    }),
+  ),
+  serviceGeography: cited(z.string().min(1)).nullable(),
+  knownFunders: z.array(
+    z.object({ name: z.string().min(1), sourceUrl: z.string().nullable() }),
+  ),
+  staff: z.array(
+    z.object({
+      name: z.string().min(1),
+      role: z.string().nullable(),
+      sourceUrl: z.string().nullable(),
+    }),
+  ),
+  recentNews: z.array(
+    z.object({
+      headline: z.string().min(1),
+      url: z.string().nullable(),
+      date: z.string().nullable(),
+    }),
+  ),
+  /** Model's overall 0–1 confidence that the profile describes the right org accurately. */
+  overallConfidence: z.number().min(0).max(1),
+});
+export type ResearchedOrgProfile = z.infer<typeof ResearchedOrgProfileSchema>;
+
+/** Plain JSON Schema mirror of ResearchedOrgProfileSchema for Gemini's responseJsonSchema. */
+export const RESEARCHED_ORG_PROFILE_JSON_SCHEMA = {
+  type: 'object',
+  properties: {
+    identityConfirmed: { type: 'boolean' },
+    orgWebsite: { type: ['string', 'null'] },
+    mission: {
+      type: ['object', 'null'],
+      properties: {
+        value: { type: 'string' },
+        sourceUrl: { type: ['string', 'null'] },
+      },
+      required: ['value', 'sourceUrl'],
+    },
+    programs: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          name: { type: 'string' },
+          description: { type: ['string', 'null'] },
+          populationServed: { type: ['string', 'null'] },
+          sourceUrl: { type: ['string', 'null'] },
+        },
+        required: ['name', 'description', 'populationServed', 'sourceUrl'],
+      },
+    },
+    serviceGeography: {
+      type: ['object', 'null'],
+      properties: {
+        value: { type: 'string' },
+        sourceUrl: { type: ['string', 'null'] },
+      },
+      required: ['value', 'sourceUrl'],
+    },
+    knownFunders: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          name: { type: 'string' },
+          sourceUrl: { type: ['string', 'null'] },
+        },
+        required: ['name', 'sourceUrl'],
+      },
+    },
+    staff: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          name: { type: 'string' },
+          role: { type: ['string', 'null'] },
+          sourceUrl: { type: ['string', 'null'] },
+        },
+        required: ['name', 'role', 'sourceUrl'],
+      },
+    },
+    recentNews: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          headline: { type: 'string' },
+          url: { type: ['string', 'null'] },
+          date: { type: ['string', 'null'] },
+        },
+        required: ['headline', 'url', 'date'],
+      },
+    },
+    overallConfidence: { type: 'number', minimum: 0, maximum: 1 },
+  },
+  required: [
+    'identityConfirmed',
+    'orgWebsite',
+    'mission',
+    'programs',
+    'serviceGeography',
+    'knownFunders',
+    'staff',
+    'recentNews',
+    'overallConfidence',
+  ],
+} as const;
+
+export interface ProfileResearchTarget {
+  readonly name: string;
+  readonly city: string | null;
+  readonly state: string;
+  readonly ein: string | null;
+  readonly nteeCode: string | null;
+}
+
+/** Prompt for the grounded research pass (Google Search tool enabled). */
+export function buildResearchPrompt(org: ProfileResearchTarget): string {
+  const where = [org.city, org.state].filter(Boolean).join(', ');
+  return [
+    `Research the nonprofit organization "${org.name}" based in ${where}.`,
+    org.ein != null ? `Its federal EIN is ${org.ein}.` : '',
+    '',
+    'Find and report, with specifics:',
+    '1. Their mission, in their own words if possible.',
+    '2. Their concrete programs/services — names, what each does, and who it serves.',
+    '3. The geography they serve (towns/counties/statewide).',
+    '4. Foundations or agencies that fund them (funder walls, annual reports, press).',
+    '5. Key staff: executive director and anyone in a development/grants role.',
+    '6. Any news from the last two years.',
+    '',
+    'CRITICAL: verify you are describing THIS organization — the one in',
+    `${where} — not a similarly-named organization elsewhere. If you cannot`,
+    'find credible information clearly about this specific organization, say',
+    'so explicitly instead of substituting a lookalike.',
+  ]
+    .filter((line) => line !== '')
+    .join('\n');
+}
+
+/** Prompt for the structured extraction pass over the research text. */
+export function buildExtractionPrompt(
+  org: ProfileResearchTarget,
+  researchText: string,
+  sourceUrls: readonly string[],
+): string {
+  return [
+    'Extract a structured profile of the nonprofit below from the research',
+    'notes. Rules:',
+    '- Only include claims supported by the research notes.',
+    '- For each claim, cite the most relevant source URL from the list, or',
+    '  null if the claim synthesizes multiple sources.',
+    '- If the research failed to clearly identify this specific organization',
+    `  (the one in ${[org.city, org.state].filter(Boolean).join(', ')}),`,
+    '  set identityConfirmed=false and overallConfidence at or below 0.3.',
+    '- Empty arrays are fine. Never invent programs, funders, staff, or news.',
+    '',
+    `Organization: ${org.name}`,
+    '',
+    `Available source URLs:\n${sourceUrls.map((u) => `- ${u}`).join('\n') || '- (none)'}`,
+    '',
+    `Research notes:\n${researchText}`,
+  ].join('\n');
+}

packages/outreach-ai/src/index.ts

@@ -6,3 +6,5 @@ export * from './agents/org-profiler/run.js';
 export * from './agents/mission-fit-judge/schema.js';
 export * from './agents/mission-fit-judge/run.js';
 export * from './grant-embedding-text.js';
+export * from './agents/org-profiler/researched-profile.js';
+export * from './agents/org-profiler/research.js';

packages/outreach-core/src/matches/queries/get-match-detail.server.ts

@@ -24,6 +24,11 @@ export interface MatchDetail {
     registrationNumber: string | null;
     missionStatement: string | null;
     profileConfidence: number | null;
+    programs: unknown;
+    knownFunders: unknown;
+    staff: unknown;
+    serviceGeography: string | null;
+    profileSources: unknown;
   };
   grant: {
     id: string;
@@ -103,6 +108,11 @@ export async function serverGetMatchDetail(
     .select({
       missionStatement: schema.orgProfiles.missionStatement,
       confidence: schema.orgProfiles.confidence,
+      programs: schema.orgProfiles.programs,
+      knownFunders: schema.orgProfiles.knownFunders,
+      staff: schema.orgProfiles.staff,
+      serviceGeography: schema.orgProfiles.serviceGeography,
+      sources: schema.orgProfiles.sources,
     })
     .from(schema.orgProfiles)
     .where(eq(schema.orgProfiles.orgId, row.orgId))
@@ -184,6 +194,11 @@ export async function serverGetMatchDetail(
       registrationNumber: row.orgRegNo,
       missionStatement: profiles[0]?.missionStatement ?? null,
       profileConfidence: profiles[0]?.confidence ?? null,
+      programs: profiles[0]?.programs ?? null,
+      knownFunders: profiles[0]?.knownFunders ?? null,
+      staff: profiles[0]?.staff ?? null,
+      serviceGeography: profiles[0]?.serviceGeography ?? null,
+      profileSources: profiles[0]?.sources ?? null,
     },
     grant: {
       id: row.grantId,

packages/outreach-core/src/orgs/actions/apply-researched-profile.server.ts

@@ -0,0 +1,57 @@
+import { sql } from 'drizzle-orm';
+
+import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
+import { schema } from '#~/db/db.js';
+
+export interface ResearchedProfileInput {
+  readonly orgId: string;
+  readonly missionStatement: string | null;
+  readonly programs: unknown;
+  readonly serviceGeography: string | null;
+  readonly recentNews: unknown;
+  readonly knownFunders: unknown;
+  readonly staff: unknown;
+  readonly sources: unknown;
+  readonly confidence: number;
+  /** 1536-dim RETRIEVAL_QUERY embedding of the researched profile text. */
+  readonly profileEmbedding: number[];
+}
+
+/**
+ * Applies a Stage 4 researched profile, OVERWRITING the row — this is the
+ * upgrade path the v0 NTEE stub (confidence 0.2, never-overwriting) exists
+ * to be replaced by. Re-running research refreshes in place.
+ */
+export async function serverApplyResearchedProfile(
+  db: NpOutreachDatabase | NpOutreachTransaction,
+  profile: ResearchedProfileInput,
+): Promise<void> {
+  await db
+    .insert(schema.orgProfiles)
+    .values({
+      orgId: profile.orgId,
+      missionStatement: profile.missionStatement,
+      programs: profile.programs,
+      serviceGeography: profile.serviceGeography,
+      recentNews: profile.recentNews,
+      knownFunders: profile.knownFunders,
+      staff: profile.staff,
+      sources: profile.sources,
+      confidence: profile.confidence,
+      profileEmbedding: profile.profileEmbedding,
+    })
+    .onConflictDoUpdate({
+      target: schema.orgProfiles.orgId,
+      set: {
+        missionStatement: sql`excluded.mission_statement`,
+        programs: sql`excluded.programs`,
+        serviceGeography: sql`excluded.service_geography`,
+        recentNews: sql`excluded.recent_news`,
+        knownFunders: sql`excluded.known_funders`,
+        staff: sql`excluded.staff`,
+        sources: sql`excluded.sources`,
+        confidence: sql`excluded.confidence`,
+        profileEmbedding: sql`excluded.profile_embedding`,
+      },
+    });
+}

packages/outreach-core/src/orgs/actions/index.server.ts

@@ -2,3 +2,4 @@ export * from './insert-org.server.js';
 export * from './upsert-org-from-registry.server.js';
 export * from './enrich-org.server.js';
 export * from './ensure-org-profile-stub.server.js';
+export * from './apply-researched-profile.server.js';

packages/outreach-core/src/orgs/queries/index.server.ts

@@ -1,3 +1,4 @@
 export * from './list-orgs-in-icp-band.server.js';
 export * from './list-orgs-needing-enrichment.server.js';
 export * from './list-match-candidate-orgs.server.js';
+export * from './list-orgs-needing-profile.server.js';

packages/outreach-core/src/orgs/queries/list-orgs-needing-profile.server.ts

@@ -0,0 +1,55 @@
+import { sql } from 'drizzle-orm';
+
+import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
+
+export interface OrgNeedingProfile {
+  id: string;
+  name: string;
+  city: string | null;
+  state: string;
+  ein: string | null;
+  nteeCode: string | null;
+  bestMatchScore: number;
+}
+
+/** Profiles below/at this confidence are NTEE-derived stubs, not research. */
+export const STUB_CONFIDENCE_CEILING = 0.2;
+
+/**
+ * The plan's cost-control rule made concrete: research only orgs that
+ * already have at least one non-rejected easy-win match (a plausible
+ * lead), best-scoring orgs first, and only when the current profile is a
+ * stub (or missing). Re-research after review feedback comes later.
+ */
+export async function serverListOrgsNeedingProfile(
+  db: NpOutreachDatabase | NpOutreachTransaction,
+  { limit }: { limit: number },
+): Promise<OrgNeedingProfile[]> {
+  const result = await db.execute(sql`
+    SELECT
+      o.id, o.name, o.city, o.state, o.ein, o.ntee_code,
+      max(m.total_score) AS best_match_score
+    FROM orgs o
+    JOIN matches m ON m.org_id = o.id
+      AND m.review_status != 'rejected'
+      AND m.easy_win = true
+    LEFT JOIN org_profiles p ON p.org_id = o.id
+    -- confidence is a float4: 0.2 stores as 0.20000000298, so a plain
+    -- <= 0.2 comparison (float8) excludes every stub. Epsilon absorbs it.
+    WHERE p.id IS NULL OR p.confidence IS NULL OR p.confidence <= ${STUB_CONFIDENCE_CEILING} + 0.001
+    GROUP BY o.id
+    ORDER BY max(m.total_score) DESC
+    LIMIT ${limit}
+  `);
+
+  const { rows } = result as unknown as { rows: Record<string, unknown>[] };
+  return rows.map((r) => ({
+    id: r.id as string,
+    name: r.name as string,
+    city: (r.city as string) ?? null,
+    state: r.state as string,
+    ein: (r.ein as string) ?? null,
+    nteeCode: (r.ntee_code as string) ?? null,
+    bestMatchScore: Number(r.best_match_score),
+  }));
+}