feat(profiler): Stage 4 grounded research profiler — cited profiles replace NTEE stubs

profileOrgs (nightly 05:45, 25/run, easy-win orgs best-first per the
plan's coarse-match-gates-profiling rule): two-pass Gemini flash —
Google Search-grounded research (groundingMetadata = citation universe)
then JSON-schema extraction citing only from it. Overwrites the stub
profile (mission/programs/geography/funders/staff/news/sources/
confidence); re-embeds mission fit from researched text unless identity
unconfirmed or confidence <0.5 (then the NTEE embedding stays —
unverified research must not steer scoring). Detail page renders
researched programs w/ source links, funders, staff.

Fixes: org_profiles.confidence is float4 — 0.2 stores as 0.20000000298
so 'confidence <= 0.2' excluded every stub (epsilon comparison); RR7
loader serialization turns unknown into never (hoisted casts).

Live: 55 orgs profiled (41 research-grade, 2 low-confidence, 0 failed).
Scoring effect verified both directions: Annie's Angels x NIH mammalian
models fit 19->12; Seacoast Pathways x Foundation for Seacoast Health
fit 25/30 — and the researched knownFunders independently names that
funder. 161 tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/outreach-worker/src/main.ts

@@ -38,6 +38,7 @@ import { setIngestGrantsDeps } from './workflows/ingest-grants.js';
 import { setIngest990pfDeps } from './workflows/ingest-990pf.js';
 import { setIngestNhdojOrgsDeps } from './workflows/ingest-nhdoj-orgs.js';
 import { setMatchGrantsDeps } from './workflows/match-grants.js';
+import { setProfileOrgsDeps } from './workflows/profile-orgs.js';
 import { setIngestPndRssDeps } from './workflows/ingest-pnd-rss.js';
 
 if (process.env.DATABASE_URL == null) {
@@ -73,6 +74,7 @@ async function main() {
   setMatchGrantsDeps({ db });
   setIngest990pfDeps({ db });
   setFederalPrecedentDeps({ db });
+  setProfileOrgsDeps({ db });
 
   DBOS.setConfig({
     name: 'helmdocs-outreach-worker',

apps/outreach-worker/src/run-once.ts

@@ -48,6 +48,10 @@ import {
   runMatchGrantsNow,
   setMatchGrantsDeps,
 } from './workflows/match-grants.js';
+import {
+  runProfileOrgsNow,
+  setProfileOrgsDeps,
+} from './workflows/profile-orgs.js';
 import {
   runIngestPndRssNow,
   setIngestPndRssDeps,
@@ -63,6 +67,7 @@ const RUNNERS: Record<string, () => Promise<void>> = {
   matchGrants: runMatchGrantsNow,
   ingest990pf: runIngest990PfNow,
   federalPrecedent: runFederalPrecedentNow,
+  profileOrgs: runProfileOrgsNow,
 };
 
 const FIRST_RUN_ORDER = [
@@ -75,6 +80,7 @@ const FIRST_RUN_ORDER = [
   'federalPrecedent',
   'embedGrants',
   'matchGrants',
+  'profileOrgs',
 ];
 
 if (process.env.DATABASE_URL == null) {
@@ -113,6 +119,7 @@ async function main() {
   setMatchGrantsDeps({ db });
   setIngest990pfDeps({ db });
   setFederalPrecedentDeps({ db });
+  setProfileOrgsDeps({ db });
 
   DBOS.setConfig({
     name: 'helmdocs-outreach-worker',

apps/outreach-worker/src/workflows/profile-orgs.ts

@@ -0,0 +1,208 @@
+/**
+ * Nightly Stage 4 profiling workflow — grounded web research for orgs
+ * that already have a plausible lead (>=1 non-rejected easy-win match),
+ * best-scoring first, replacing their NTEE-stub profiles with cited,
+ * researched ones. Better profiles → better mission-fit embeddings on the
+ * NEXT match run → better personalization at review time.
+ *
+ * The nightly batch cap bounds LLM spend (two Flash calls + one embedding
+ * per org, ≈ cents each); env `PROFILE_ORGS_PER_RUN` overrides.
+ *
+ * Low-confidence results (identity not confirmed, or the model reports
+ * <= 0.4) are STILL stored — with their low confidence — so the review UI
+ * can warn, but they do NOT overwrite... actually they do overwrite the
+ * stub, which is correct: "we looked and couldn't confirm" is more
+ * information than an NTEE guess, and the confidence field carries it.
+ * The profile embedding, however, keeps mission-fit honest: identity-
+ * unconfirmed profiles are embedded from the stub text, not the
+ * possibly-wrong research.
+ *
+ * Runs at 05:45, after matchGrants (05:15). Registration follows
+ * `ingest-grants.ts` exactly.
+ */
+import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
+import {
+  buildProfileEmbeddingText,
+  generateQueryEmbedding,
+  researchOrgProfile,
+  type ResearchedProfileResult,
+} from '@novelpad/outreach-ai';
+import { buildOrgMissionText, type schema } from '@novelpad/outreach-core';
+import {
+  serverApplyResearchedProfile,
+  serverListOrgsNeedingProfile,
+  type OrgNeedingProfile,
+} from '@novelpad/outreach-core/server';
+import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
+
+export type OutreachDb = NodePgDatabase<typeof schema>;
+
+const DEFAULT_ORGS_PER_RUN = 25;
+/** Below this, the research embedding is not trusted for mission fit. */
+const EMBEDDING_TRUST_FLOOR = 0.5;
+
+export interface ProfileOrgsDeps {
+  readonly db: OutreachDb;
+}
+
+let registeredDeps: ProfileOrgsDeps | null = null;
+
+export function setProfileOrgsDeps(deps: ProfileOrgsDeps): void {
+  registeredDeps = deps;
+}
+
+function getProfileOrgsDeps(): ProfileOrgsDeps {
+  if (registeredDeps == null) {
+    throw new Error(
+      'ProfileOrgsDeps not registered. Call setProfileOrgsDeps() before DBOS.launch().',
+    );
+  }
+  return registeredDeps;
+}
+
+async function listOrgsNeedingProfile(
+  db: OutreachDb,
+  limit: number,
+): Promise<OrgNeedingProfile[]> {
+  return serverListOrgsNeedingProfile(db, { limit });
+}
+const listOrgsNeedingProfileStep = DBOS.registerStep(listOrgsNeedingProfile, {
+  name: 'listOrgsNeedingProfile',
+  retriesAllowed: true,
+  maxAttempts: 3,
+});
+
+async function researchOrg(
+  org: OrgNeedingProfile,
+): Promise<ResearchedProfileResult> {
+  return researchOrgProfile(org);
+}
+const researchOrgStep = DBOS.registerStep(researchOrg, {
+  name: 'researchOrgProfile',
+  retriesAllowed: true,
+  maxAttempts: 2,
+});
+
+async function storeProfile(
+  db: OutreachDb,
+  org: OrgNeedingProfile,
+  result: ResearchedProfileResult,
+): Promise<void> {
+  const { profile, sources } = result;
+
+  const trustworthy =
+    profile.identityConfirmed &&
+    profile.overallConfidence >= EMBEDDING_TRUST_FLOOR;
+  const embeddingText = trustworthy
+    ? buildProfileEmbeddingText(org, profile)
+    : buildOrgMissionText(org);
+  const embedding = await generateQueryEmbedding(embeddingText);
+
+  await serverApplyResearchedProfile(db, {
+    orgId: org.id,
+    missionStatement: profile.mission?.value ?? null,
+    programs: profile.programs,
+    serviceGeography: profile.serviceGeography?.value ?? null,
+    recentNews: profile.recentNews,
+    knownFunders: profile.knownFunders,
+    staff: profile.staff,
+    sources,
+    confidence: profile.identityConfirmed
+      ? profile.overallConfidence
+      : Math.min(profile.overallConfidence, 0.3),
+    profileEmbedding: embedding,
+  });
+}
+const storeProfileStep = DBOS.registerStep(storeProfile, {
+  name: 'storeResearchedProfile',
+  retriesAllowed: true,
+  maxAttempts: 3,
+});
+
+async function runProfileOrgs(): Promise<void> {
+  const { db } = getProfileOrgsDeps();
+
+  const limit = Number(process.env.PROFILE_ORGS_PER_RUN ?? DEFAULT_ORGS_PER_RUN);
+  const orgs = await listOrgsNeedingProfileStep(db, limit);
+  if (orgs.length === 0) {
+    console.log('[profile-orgs] no easy-win orgs awaiting research');
+    return;
+  }
+
+  let profiled = 0;
+  let lowConfidence = 0;
+  let failed = 0;
+  for (const org of orgs) {
+    try {
+      const result = await researchOrgStep(org);
+      await storeProfileStep(db, org, result);
+      profiled++;
+      if (
+        !result.profile.identityConfirmed ||
+        result.profile.overallConfidence < EMBEDDING_TRUST_FLOOR
+      ) {
+        lowConfidence++;
+      }
+    } catch (err) {
+      failed++;
+      console.error(`[profile-orgs] org "${org.name}" failed:`, err);
+    }
+  }
+
+  console.log(
+    `[profile-orgs] attempted=${orgs.length} profiled=${profiled} lowConfidence=${lowConfidence} failed=${failed}`,
+  );
+  if (failed > 0 && failed / orgs.length > 0.2) {
+    throw new Error(
+      `[profile-orgs] systemic failure: ${failed}/${orgs.length} orgs failed`,
+    );
+  }
+}
+
+const g = globalThis as unknown as {
+  __outreachProfileOrgsRegistered?: boolean;
+  __outreachProfileOrgsHandle?: (
+    scheduledTime: Date,
+    startedAt: Date,
+  ) => Promise<void>;
+};
+
+if (!g.__outreachProfileOrgsRegistered) {
+  g.__outreachProfileOrgsRegistered = true;
+
+  const profileOrgs = async (_scheduledTime: Date, _startedAt: Date) => {
+    try {
+      await runProfileOrgs();
+    } catch (err) {
+      console.error('[profile-orgs] pass failed:', err);
+      throw err;
+    }
+  };
+
+  // Must be registered as BOTH a workflow and a scheduled function,
+  // referencing the same function object — see ingest-grants.ts.
+  g.__outreachProfileOrgsHandle = DBOS.registerWorkflow(profileOrgs, {
+    name: 'profileOrgs',
+  });
+  DBOS.registerScheduled(profileOrgs, {
+    crontab: '45 5 * * *',
+    name: 'profileOrgs',
+    mode: SchedulerMode.ExactlyOncePerInterval,
+  });
+}
+
+/**
+ * Starts one durable run of this workflow immediately through DBOS —
+ * the exact production path (workflow + checkpointed steps), used by
+ * `run-once.ts` for supervised/manual passes. Requires deps injected and
+ * `DBOS.launch()` completed.
+ */
+export function runProfileOrgsNow(): Promise<void> {
+  const handle = g.__outreachProfileOrgsHandle;
+  if (handle == null) {
+    throw new Error(
+      'profileOrgs is not registered; was this module imported before DBOS.launch()?',
+    );
+  }
+  return handle(new Date(), new Date());
+}