feat(scoring): nightly grant-embedding workflow (Stage 2, step 1)
embedGrants (cron 04:15, after the ingest crons): open grants with a
synopsis and no vector → buildGrantEmbeddingText (pure, tested:
title+funder+program areas+synopsis, 8K cap) → gemini-embedding-001 @
1536 dims RETRIEVAL_DOCUMENT (org profiles will embed as
RETRIEVAL_QUERY on the other side) → grants.synopsis_embedding.
Chunked embed→store (100/chunk) so failures resume from the last
stored chunk; 500/run spend cap.
Core: serverListGrantsNeedingEmbedding (open+unembedded, closest
deadline first), serverSetGrantEmbeddings. Worker gains
@novelpad/outreach-ai dep; wired into main.ts and run-once (incl. the
missed run-once deps injection).
Live-verified: 199/199 open grants embedded in 16s; semantic probe
('after-school STEM education for youth') ranks NCI Youth Enjoy
Science R25 first at 0.639 cosine via the hnsw index.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,2 +1,3 @@
|
||||
export * from './expire-closed-grants.server.js';
|
||||
export * from './insert-grants.server.js';
|
||||
export * from './set-grant-embeddings.server.js';
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { eq, sql } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
export interface GrantEmbedding {
|
||||
readonly grantId: string;
|
||||
/** 1536-dim vector from gemini-embedding-001 (RETRIEVAL_DOCUMENT). */
|
||||
readonly embedding: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores synopsis embeddings for a batch of grants. Plain sequential
|
||||
* updates — batches are small (≤100) and this runs inside a nightly job,
|
||||
* so a multi-row VALUES join isn't worth the SQL gymnastics yet.
|
||||
*/
|
||||
export async function serverSetGrantEmbeddings(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
embeddings: ReadonlyArray<GrantEmbedding>,
|
||||
): Promise<void> {
|
||||
for (const { grantId, embedding } of embeddings) {
|
||||
await db
|
||||
.update(schema.grants)
|
||||
.set({ synopsisEmbedding: embedding, updatedAt: sql`now()` })
|
||||
.where(eq(schema.grants.id, grantId));
|
||||
}
|
||||
}
|
||||
@@ -1 +1,2 @@
|
||||
export * from './list-open-grants.server.js';
|
||||
export * from './list-grants-needing-embedding.server.js';
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm';
|
||||
|
||||
import type { NpOutreachDatabase, NpOutreachTransaction } from '#~/db/db.js';
|
||||
import { schema } from '#~/db/db.js';
|
||||
|
||||
export interface GrantNeedingEmbedding {
|
||||
id: string;
|
||||
funder: string;
|
||||
title: string;
|
||||
synopsis: string | null;
|
||||
programAreas: string[] | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open grants that have text to embed but no synopsis embedding yet.
|
||||
* Closed/expired grants are skipped — they can't be matched, so embedding
|
||||
* them is wasted spend; if one re-opens, its upsert clears nothing, and it
|
||||
* was embedded while open anyway.
|
||||
*/
|
||||
export async function serverListGrantsNeedingEmbedding(
|
||||
db: NpOutreachDatabase | NpOutreachTransaction,
|
||||
{ limit }: { limit: number },
|
||||
): Promise<GrantNeedingEmbedding[]> {
|
||||
return db
|
||||
.select({
|
||||
id: schema.grants.id,
|
||||
funder: schema.grants.funder,
|
||||
title: schema.grants.title,
|
||||
synopsis: schema.grants.synopsis,
|
||||
programAreas: schema.grants.programAreas,
|
||||
})
|
||||
.from(schema.grants)
|
||||
.where(
|
||||
and(
|
||||
eq(schema.grants.status, 'open'),
|
||||
isNull(schema.grants.synopsisEmbedding),
|
||||
isNotNull(schema.grants.synopsis),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(schema.grants.closeDate))
|
||||
.limit(limit);
|
||||
}
|
||||
Reference in New Issue
Block a user