feat: scaffold outreach engine monorepo on the novelpad-desktop stack
Workspaces: config (copied), outreach-core (schema + actions/queries + hard gates), outreach-ai (Gemini client + embeddings copies, profiler and mission-fit-judge agent stubs), outreach-worker (DBOS executor with nightly ingest + hourly expiry workflows), outreach-review (RR7 review queue v0). Initial drizzle migration incl. pgvector extension. Stack contract: Yarn 4.5.0 + Turbo, Node 22.16, Drizzle 0.44.6 + pgvector, DBOS 4.17.6, @google/genai on Vertex, gemini-embedding-001 @1536, React Router v7. Files copied from novelpad-desktop carry provenance headers @ 62c56b87. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
53
apps/outreach-review/README.md
Normal file
53
apps/outreach-review/README.md
Normal file
@@ -0,0 +1,53 @@
|
||||
# @novelpad/outreach-review
|
||||
|
||||
Human review queue for (org, grant) matches produced by the scoring pipeline in `packages/outreach-core`. A reviewer sees each pending match — organization, hero grant, total score, easy-win flag — and approves or rejects it. Approved matches are picked up downstream and synced into Apollo sequences.
|
||||
|
||||
## Auth
|
||||
|
||||
**There is no authentication in this app, by design.** It is a tiny internal tool intended to run behind:
|
||||
|
||||
- Google IAP (Identity-Aware Proxy) in any hosted environment, or
|
||||
- `localhost` only, for local development
|
||||
|
||||
Do **not** add `better-auth` (or any other auth library) to this workspace. If this app ever needs to leave an IAP-fronted network, that's a deliberate follow-up decision, not something to bolt on ad hoc here.
|
||||
|
||||
## Development
|
||||
|
||||
```
|
||||
yarn dev # react-router dev
|
||||
yarn build # react-router build
|
||||
yarn start # serve the production build
|
||||
yarn typecheck # react-router typegen && tsc --noEmit
|
||||
```
|
||||
|
||||
Requires `DATABASE_URL` pointing at the outreach Postgres instance (see `packages/outreach-core`). Optional: `PG_POOL_MAX`, `PG_CONNECTION_TIMEOUT_MS`.
|
||||
|
||||
## Data contract
|
||||
|
||||
This app is a thin consumer of `@novelpad/outreach-core`. It assumes the following exports (server-side query/action functions live under the `./server` subpath, matching the `packages/core` convention in novelpad-desktop):
|
||||
|
||||
```ts
|
||||
// @novelpad/outreach-core
|
||||
export const schema: /* drizzle schema */;
|
||||
export type NpOutreachDatabase = /* PgDatabase<..., typeof schema> */;
|
||||
export interface PendingReviewMatch {
|
||||
id: string;
|
||||
orgName: string;
|
||||
heroGrantTitle: string;
|
||||
totalScore: number;
|
||||
isEasyWin: boolean;
|
||||
}
|
||||
export type MatchReviewDecision = 'approved' | 'rejected';
|
||||
|
||||
// @novelpad/outreach-core/server
|
||||
export function serverListPendingReviewMatches(
|
||||
db: NpOutreachDatabase,
|
||||
): Promise<PendingReviewMatch[]>;
|
||||
export function serverSetMatchReview(
|
||||
db: NpOutreachDatabase,
|
||||
matchId: string,
|
||||
decision: MatchReviewDecision,
|
||||
): Promise<void>;
|
||||
```
|
||||
|
||||
If `packages/outreach-core`'s matches domain (`src/matches/queries`, `src/matches/actions`) lands with different names or a different `PendingReviewMatch` shape, update `app/db.server.ts` and `app/routes/_index.tsx` to match rather than reshaping data client-side.
|
||||
7
apps/outreach-review/app/assets/main.css
Normal file
7
apps/outreach-review/app/assets/main.css
Normal file
@@ -0,0 +1,7 @@
|
||||
/* Deliberately font-free (see root.tsx) — this app doesn't pull in
|
||||
@novelpad/config/shadcn.css because that file's first several lines are
|
||||
@fontsource imports. Keeping the review queue on system fonts avoids
|
||||
shipping the full novelpad-desktop font set for a tiny internal tool. */
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
33
apps/outreach-review/app/db.server.ts
Normal file
33
apps/outreach-review/app/db.server.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
// Copied/adapted from novelpad-desktop apps/website/app/db/db.server.ts @ 62c56b87
|
||||
import { schema, type NpOutreachDatabase } from '@novelpad/outreach-core';
|
||||
import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
import pkg from 'pg';
|
||||
|
||||
const { Pool } = pkg;
|
||||
|
||||
// HMR-safe singleton. Without this, Vite re-evaluates this module on hot
|
||||
// reload and instantiates a fresh Pool, while the previous pool lingers with
|
||||
// open connections until idleTimeoutMillis fires — see the equivalent note
|
||||
// in novelpad-desktop's apps/website/app/db/db.server.ts.
|
||||
const g = globalThis as unknown as {
|
||||
__outreachPool?: InstanceType<typeof Pool>;
|
||||
__outreachDb?: NpOutreachDatabase;
|
||||
};
|
||||
|
||||
const pool =
|
||||
g.__outreachPool ??
|
||||
(g.__outreachPool = new Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
application_name: 'outreach-review',
|
||||
max: Number(process.env.PG_POOL_MAX ?? 10),
|
||||
idleTimeoutMillis: 30000,
|
||||
connectionTimeoutMillis: Number(
|
||||
process.env.PG_CONNECTION_TIMEOUT_MS ?? 15000,
|
||||
),
|
||||
}));
|
||||
|
||||
const db =
|
||||
g.__outreachDb ??
|
||||
(g.__outreachDb = drizzle(pool, { schema }) as unknown as NpOutreachDatabase);
|
||||
|
||||
export { db, pool };
|
||||
56
apps/outreach-review/app/root.tsx
Normal file
56
apps/outreach-review/app/root.tsx
Normal file
@@ -0,0 +1,56 @@
|
||||
import { isRouteErrorResponse, Links, Meta, Outlet, Scripts } from 'react-router';
|
||||
|
||||
import type { Route } from './+types/root.js';
|
||||
import appCssUrl from './assets/main.css?url';
|
||||
|
||||
export function Layout({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charSet="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>HelmDocs Outreach Review</title>
|
||||
<link rel="stylesheet" href={appCssUrl} />
|
||||
<Meta />
|
||||
<Links />
|
||||
</head>
|
||||
<body>
|
||||
{children}
|
||||
<Scripts />
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
|
||||
export default function App() {
|
||||
return <Outlet />;
|
||||
}
|
||||
|
||||
export function ErrorBoundary({ error }: Route.ErrorBoundaryProps) {
|
||||
let message = 'Oops!';
|
||||
let details = 'An unexpected error occurred.';
|
||||
let stack: string | undefined;
|
||||
|
||||
if (isRouteErrorResponse(error)) {
|
||||
message = error.status === 404 ? '404' : 'Error';
|
||||
details =
|
||||
error.status === 404
|
||||
? 'The requested page could not be found.'
|
||||
: error.statusText || details;
|
||||
} else if (import.meta.env.DEV && error && error instanceof Error) {
|
||||
details = error.message;
|
||||
stack = error.stack;
|
||||
}
|
||||
|
||||
return (
|
||||
<main className="pt-16 p-4 container mx-auto">
|
||||
<h1>{message}</h1>
|
||||
<p>{details}</p>
|
||||
{stack && (
|
||||
<pre className="w-full p-4 overflow-x-auto">
|
||||
<code>{stack}</code>
|
||||
</pre>
|
||||
)}
|
||||
</main>
|
||||
);
|
||||
}
|
||||
3
apps/outreach-review/app/routes.ts
Normal file
3
apps/outreach-review/app/routes.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
import { index, type RouteConfig } from '@react-router/dev/routes';
|
||||
|
||||
export default [index('routes/_index.tsx')] satisfies RouteConfig;
|
||||
109
apps/outreach-review/app/routes/_index.tsx
Normal file
109
apps/outreach-review/app/routes/_index.tsx
Normal file
@@ -0,0 +1,109 @@
|
||||
import {
|
||||
serverListPendingReviewMatches,
|
||||
serverSetMatchReview,
|
||||
} from '@novelpad/outreach-core/server';
|
||||
import { Form } from 'react-router';
|
||||
|
||||
import { db } from '#~/db.server.js';
|
||||
|
||||
import type { Route } from './+types/_index.js';
|
||||
|
||||
export async function loader({ request }: Route.LoaderArgs) {
|
||||
if (request.method === 'HEAD') return;
|
||||
|
||||
const matches = await serverListPendingReviewMatches(db);
|
||||
return { matches };
|
||||
}
|
||||
|
||||
export async function action({ request }: Route.ActionArgs) {
|
||||
if (request.method === 'HEAD') return;
|
||||
|
||||
const formData = await request.formData();
|
||||
const matchId = String(formData.get('matchId') ?? '');
|
||||
const decision = formData.get('decision');
|
||||
|
||||
if (!matchId || (decision !== 'approved' && decision !== 'rejected')) {
|
||||
throw new Response('Invalid review submission', { status: 400 });
|
||||
}
|
||||
|
||||
await serverSetMatchReview(db, {
|
||||
matchId,
|
||||
reviewStatus: decision,
|
||||
rejectReason: decision === 'rejected' ? 'other' : undefined,
|
||||
});
|
||||
return { ok: true as const };
|
||||
}
|
||||
|
||||
export default function ReviewQueue({ loaderData }: Route.ComponentProps) {
|
||||
const matches = loaderData?.matches ?? [];
|
||||
|
||||
return (
|
||||
<main className="container mx-auto p-6">
|
||||
<h1 className="mb-4 text-2xl font-semibold">Grant Match Review Queue</h1>
|
||||
|
||||
{matches.length === 0 ? (
|
||||
<p role="status" className="text-gray-600">
|
||||
No pending matches to review.
|
||||
</p>
|
||||
) : (
|
||||
<table className="w-full border-collapse text-left">
|
||||
<thead>
|
||||
<tr className="border-b">
|
||||
<th scope="col" className="p-2">
|
||||
Organization
|
||||
</th>
|
||||
<th scope="col" className="p-2">
|
||||
Grant
|
||||
</th>
|
||||
<th scope="col" className="p-2">
|
||||
Score
|
||||
</th>
|
||||
<th scope="col" className="p-2">
|
||||
Easy Win
|
||||
</th>
|
||||
<th scope="col" className="p-2">
|
||||
Actions
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{matches.map((match) => (
|
||||
<tr key={match.id} className="border-b">
|
||||
<td className="p-2">{match.orgName}</td>
|
||||
<td className="p-2">{match.grantTitle}</td>
|
||||
<td className="p-2">{match.totalScore}</td>
|
||||
<td className="p-2">{match.easyWin ? 'Yes' : 'No'}</td>
|
||||
<td className="p-2">
|
||||
<div className="flex gap-2">
|
||||
<Form method="post">
|
||||
<input type="hidden" name="matchId" value={match.id} />
|
||||
<input type="hidden" name="decision" value="approved" />
|
||||
<button
|
||||
type="submit"
|
||||
aria-label={`Approve match for ${match.orgName}`}
|
||||
className="rounded bg-green-600 px-3 py-1 text-white hover:bg-green-700"
|
||||
>
|
||||
Approve
|
||||
</button>
|
||||
</Form>
|
||||
<Form method="post">
|
||||
<input type="hidden" name="matchId" value={match.id} />
|
||||
<input type="hidden" name="decision" value="rejected" />
|
||||
<button
|
||||
type="submit"
|
||||
aria-label={`Reject match for ${match.orgName}`}
|
||||
className="rounded bg-red-600 px-3 py-1 text-white hover:bg-red-700"
|
||||
>
|
||||
Reject
|
||||
</button>
|
||||
</Form>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</main>
|
||||
);
|
||||
}
|
||||
40
apps/outreach-review/package.json
Normal file
40
apps/outreach-review/package.json
Normal file
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"name": "@novelpad/outreach-review",
|
||||
"packageManager": "yarn@4.5.0",
|
||||
"type": "module",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "react-router dev",
|
||||
"build": "react-router build",
|
||||
"start": "react-router-serve ./build/server/index.js",
|
||||
"typecheck": "react-router typegen && tsc --noEmit"
|
||||
},
|
||||
"imports": {
|
||||
"#~/*": "./app/*"
|
||||
},
|
||||
"dependencies": {
|
||||
"@novelpad/outreach-core": "workspace:^",
|
||||
"@react-router/node": "^7.9.1",
|
||||
"@react-router/serve": "^7.9.1",
|
||||
"drizzle-orm": "0.44.6",
|
||||
"isbot": "4",
|
||||
"pg": "8.20.0",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"react-router": "^7.9.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@novelpad/config": "workspace:^",
|
||||
"@react-router/dev": "^7.9.1",
|
||||
"@types/node": "^22",
|
||||
"@types/pg": "8.20.0",
|
||||
"@types/react": "^19.0.9",
|
||||
"@types/react-dom": "^19.0.3",
|
||||
"autoprefixer": "^10.4.20",
|
||||
"postcss": "^8.4.47",
|
||||
"tailwindcss": "^3.4.13",
|
||||
"typescript": "^5.9.3",
|
||||
"vite": "^5.4.9"
|
||||
}
|
||||
}
|
||||
5
apps/outreach-review/react-router.config.ts
Normal file
5
apps/outreach-review/react-router.config.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
import type { Config } from '@react-router/dev/config';
|
||||
|
||||
export default {
|
||||
ssr: true,
|
||||
} satisfies Config;
|
||||
7
apps/outreach-review/tailwind.config.ts
Normal file
7
apps/outreach-review/tailwind.config.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
import tailwindConfig from '@novelpad/config/tailwind.config.js';
|
||||
import type { Config } from 'tailwindcss';
|
||||
|
||||
export default {
|
||||
...tailwindConfig,
|
||||
content: ['./app/**/*.{js,ts,jsx,tsx}'],
|
||||
} satisfies Config;
|
||||
28
apps/outreach-review/tsconfig.json
Normal file
28
apps/outreach-review/tsconfig.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"extends": "@novelpad/config/tsconfig.base.json",
|
||||
"include": [
|
||||
"app/**/*.ts",
|
||||
"app/**/*.tsx",
|
||||
".react-router/types/**/*"
|
||||
],
|
||||
"exclude": ["node_modules", "build"],
|
||||
"compilerOptions": {
|
||||
"lib": ["DOM", "DOM.Iterable", "ES2022"],
|
||||
"types": ["node", "vite/client"],
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "bundler",
|
||||
"jsx": "react-jsx",
|
||||
"rootDirs": [".", "./.react-router/types"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"#~/*": ["./app/*"]
|
||||
},
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": false,
|
||||
"noEmit": true,
|
||||
"resolveJsonModule": true,
|
||||
"skipLibCheck": true,
|
||||
"noUncheckedIndexedAccess": false
|
||||
}
|
||||
}
|
||||
21
apps/outreach-review/vite.config.ts
Normal file
21
apps/outreach-review/vite.config.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { reactRouter } from '@react-router/dev/vite';
|
||||
import tailwindcss from 'tailwindcss';
|
||||
import { defineConfig } from 'vite';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [reactRouter()],
|
||||
build: {
|
||||
target: 'es2022',
|
||||
sourcemap: process.env.NODE_ENV !== 'production',
|
||||
},
|
||||
ssr: {
|
||||
// pg is Node-only (uses net/tls) and should load from Node's module
|
||||
// resolver rather than Vite's SSR bundler, matching apps/website.
|
||||
external: ['pg'],
|
||||
},
|
||||
css: {
|
||||
postcss: {
|
||||
plugins: [tailwindcss()],
|
||||
},
|
||||
},
|
||||
});
|
||||
13
apps/outreach-worker/.env.example
Normal file
13
apps/outreach-worker/.env.example
Normal file
@@ -0,0 +1,13 @@
|
||||
# Postgres connection string shared by the DBOS system schema, the pg Pool,
|
||||
# and the Drizzle client over the outreach schema. Required — main.ts throws
|
||||
# on boot if this is unset.
|
||||
DATABASE_URL=postgres://postgres:postgres@localhost:5432/helmdocs_outreach
|
||||
|
||||
# Path to a GCP service account key JSON used by ingestion/scoring steps that
|
||||
# call Google-hosted APIs (e.g. @google/genai subscoring, Drive-backed org
|
||||
# profile lookups). Not read directly by main.ts yet; present here so it's
|
||||
# provisioned alongside DATABASE_URL for the workflows that will need it.
|
||||
GCP_SERVICE_ACCOUNT_KEY_PATH=./secrets/gcp-service-account.json
|
||||
|
||||
# Optional: cap the pg Pool size (defaults to 20 — see main.ts).
|
||||
# PG_POOL_MAX=20
|
||||
61
apps/outreach-worker/README.md
Normal file
61
apps/outreach-worker/README.md
Normal file
@@ -0,0 +1,61 @@
|
||||
# @novelpad/outreach-worker
|
||||
|
||||
DBOS executor process for the HelmDocs grant-match outreach engine. Runs the
|
||||
scheduled workflows that keep the outreach schema's grant catalog fresh —
|
||||
nightly ingestion from upstream sources and an hourly expiry sweep — decoupled
|
||||
from any HTTP-facing app in this monorepo (mirrors the `workflow-worker` split
|
||||
in `novelpad-desktop`).
|
||||
|
||||
## Boot order (load-bearing)
|
||||
|
||||
`src/main.ts` imports `./workflows/ingest-grants.js` and
|
||||
`./workflows/expire-grants.js` **before** calling `DBOS.launch()`. Each of
|
||||
those modules calls `DBOS.registerWorkflow` + `DBOS.registerScheduled` at
|
||||
module-evaluation time — DBOS only dispatches scheduled/queued jobs for
|
||||
functions that were registered before launch, so importing them after launch
|
||||
(or not at all) silently means the cron jobs never fire.
|
||||
|
||||
The `db` handle is then threaded into each workflow module via its
|
||||
`set*Deps` injector (`setIngestGrantsDeps` / `setExpireGrantsDeps`), also
|
||||
before launch — DBOS serializes scheduled-function arguments, so a Drizzle
|
||||
client can't be passed through the scheduler call itself. This is the same
|
||||
module-scope-registry pattern novelpad-desktop's `workflow-worker` uses for
|
||||
`setStartDeps`.
|
||||
|
||||
```
|
||||
1. import workflow modules → registers ingestGrants / expireGrants
|
||||
2. build pg Pool + drizzle(db)
|
||||
3. set*Deps({ db }) → populates each workflow's registry
|
||||
4. DBOS.setConfig(...)
|
||||
5. DBOS.launch() → scheduler starts firing
|
||||
```
|
||||
|
||||
## Workflows
|
||||
|
||||
- **`ingestGrants`** (`0 3 * * *`, nightly) — `fetchGrantsGov` (stub; real
|
||||
implementation calls the Grants.gov Search2 API,
|
||||
`POST https://api.grants.gov/v1/api/search2`) → `normalize` → upsert via
|
||||
`serverInsertGrants` from `@novelpad/outreach-core/server`. NH state
|
||||
postings and 990-PF extracts land as additional fetch+normalize steps
|
||||
later.
|
||||
- **`expireGrants`** (`0 * * * *`, hourly) — marks grants whose close date has
|
||||
passed as closed via `serverExpireClosedGrants`, so they drop out of the
|
||||
active match/scoring pool.
|
||||
|
||||
Both are registered as a DBOS workflow *and* a scheduled function referencing
|
||||
the same function object (`DBOS.registerWorkflow` then `DBOS.registerScheduled`)
|
||||
— see the doc comments in `src/workflows/*.ts` for why the dual registration
|
||||
is required.
|
||||
|
||||
## Scripts
|
||||
|
||||
- `yarn dev` — `node --env-file=.env --env-file-if-exists=.env.local --import tsx/esm ./src/main.ts`
|
||||
- `yarn build` — esbuild bundle to `build/main.js` (`--packages=external`)
|
||||
- `yarn start` — run the built bundle
|
||||
- `yarn typecheck` — `tsc --noEmit`
|
||||
|
||||
## Environment
|
||||
|
||||
Copy `.env.example` to `.env` and fill in `DATABASE_URL` (required — `main.ts`
|
||||
throws on boot without it) and `GCP_SERVICE_ACCOUNT_KEY_PATH` (used by
|
||||
future ingestion/scoring steps that call Google-hosted APIs).
|
||||
31
apps/outreach-worker/package.json
Normal file
31
apps/outreach-worker/package.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "@novelpad/outreach-worker",
|
||||
"packageManager": "yarn@4.5.0",
|
||||
"type": "module",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "node --env-file=.env --env-file-if-exists=.env.local --import tsx/esm ./src/main.ts",
|
||||
"build": "esbuild src/main.ts --bundle --platform=node --format=esm --outfile=build/main.js --packages=external",
|
||||
"start": "node ./build/main.js",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"imports": {
|
||||
"#~/*": "./src/*"
|
||||
},
|
||||
"dependencies": {
|
||||
"@dbos-inc/dbos-sdk": "4.17.6",
|
||||
"@dbos-inc/drizzle-datasource": "4.17.6",
|
||||
"@novelpad/outreach-core": "workspace:^",
|
||||
"drizzle-orm": "0.44.6",
|
||||
"pg": "8.20.0",
|
||||
"tsx": "^4.19.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@novelpad/config": "workspace:^",
|
||||
"@types/node": "^22",
|
||||
"@types/pg": "8.20.0",
|
||||
"esbuild": "^0.24.0",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
76
apps/outreach-worker/src/main.ts
Normal file
76
apps/outreach-worker/src/main.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
// Copied/adapted from novelpad-desktop apps/workflow-worker/src/main.ts @ 62c56b87
|
||||
/**
|
||||
* Outreach worker — DBOS executor process.
|
||||
*
|
||||
* Runs the grant-match outreach engine's scheduled workflows (nightly grant
|
||||
* ingestion, hourly expiry sweep, and future scoring/sync jobs) against the
|
||||
* outreach Postgres schema. Kept as its own process (mirroring novelpad's
|
||||
* workflow-worker split) so long-running ingestion/scoring steps don't share
|
||||
* a runtime with any future HTTP-facing app in this monorepo.
|
||||
*
|
||||
* Boot order is load-bearing (see ADR 0007 in novelpad-desktop):
|
||||
* 1. Import the workflow modules for side-effect registration
|
||||
* (`DBOS.registerWorkflow` / `DBOS.registerScheduled`) BEFORE
|
||||
* `DBOS.launch()` — DBOS only dispatches scheduled/queued jobs for
|
||||
* workflows that were registered before launch.
|
||||
* 2. Build the pg Pool + Drizzle db.
|
||||
* 3. `DBOS.setConfig(...)`.
|
||||
* 4. `DBOS.launch()` — register-mark closes; the scheduler starts firing.
|
||||
*/
|
||||
import { DBOS } from '@dbos-inc/dbos-sdk';
|
||||
import { DrizzleDataSource } from '@dbos-inc/drizzle-datasource';
|
||||
import { schema } from '@novelpad/outreach-core';
|
||||
import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
import pg from 'pg';
|
||||
|
||||
// Importing these modules registers `ingestGrants` (nightly) and
|
||||
// `expireGrants` (hourly) as DBOS workflows + scheduled functions as a side
|
||||
// effect of module evaluation. Must happen before `DBOS.launch()` below. Each
|
||||
// module also exports a `set*Deps` injector — DBOS serializes
|
||||
// scheduled-function args, so the `db` handle can't be passed through the
|
||||
// scheduler; it's threaded in via this module-scope registry instead (same
|
||||
// pattern as novelpad-desktop's `setStartDeps`).
|
||||
import { setExpireGrantsDeps } from './workflows/expire-grants.js';
|
||||
import { setIngestGrantsDeps } from './workflows/ingest-grants.js';
|
||||
|
||||
if (process.env.DATABASE_URL == null) {
|
||||
throw new Error('outreach-worker: DATABASE_URL is required');
|
||||
}
|
||||
|
||||
const { Pool } = pg;
|
||||
const dbosClientConfig = {
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
application_name: 'helmdocs-outreach-worker',
|
||||
} satisfies pg.ClientConfig;
|
||||
const pool = new Pool({
|
||||
...dbosClientConfig,
|
||||
max: Number(process.env.PG_POOL_MAX ?? 20),
|
||||
});
|
||||
|
||||
export const db = drizzle(pool, { schema });
|
||||
|
||||
async function main() {
|
||||
// DBOS system schema lives in this same database; initialize it with the
|
||||
// same connection settings as the worker pool.
|
||||
await DrizzleDataSource.initializeDBOSSchema(dbosClientConfig);
|
||||
|
||||
// Inject the shared `db` handle into each scheduled workflow's deps
|
||||
// registry before launch, so the first scheduled tick (which may fire
|
||||
// immediately on an `ExactlyOncePerInterval` catch-up) always has it.
|
||||
setIngestGrantsDeps({ db });
|
||||
setExpireGrantsDeps({ db });
|
||||
|
||||
DBOS.setConfig({
|
||||
name: 'helmdocs-outreach-worker',
|
||||
systemDatabasePool: pool,
|
||||
runAdminServer: false,
|
||||
});
|
||||
await DBOS.launch();
|
||||
|
||||
console.log('[outreach-worker] launched; scheduled workflows active');
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[outreach-worker] fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
70
apps/outreach-worker/src/workflows/expire-grants.ts
Normal file
70
apps/outreach-worker/src/workflows/expire-grants.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Hourly grant expiry sweep.
|
||||
*
|
||||
* Marks grants whose close date has passed as closed so they drop out of the
|
||||
* active match/scoring pool. See `ingest-grants.ts` for the registration
|
||||
* pattern this mirrors (deps-registry + dual workflow/scheduled registration).
|
||||
*/
|
||||
import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
|
||||
import type { schema } from '@novelpad/outreach-core';
|
||||
import { serverExpireClosedGrants } from '@novelpad/outreach-core/server';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
export type OutreachDb = NodePgDatabase<typeof schema>;
|
||||
|
||||
export interface ExpireGrantsDeps {
|
||||
readonly db: OutreachDb;
|
||||
}
|
||||
|
||||
let registeredDeps: ExpireGrantsDeps | null = null;
|
||||
|
||||
export function setExpireGrantsDeps(deps: ExpireGrantsDeps): void {
|
||||
registeredDeps = deps;
|
||||
}
|
||||
|
||||
function getExpireGrantsDeps(): ExpireGrantsDeps {
|
||||
if (registeredDeps == null) {
|
||||
throw new Error(
|
||||
'ExpireGrantsDeps not registered. Call setExpireGrantsDeps() before DBOS.launch().',
|
||||
);
|
||||
}
|
||||
return registeredDeps;
|
||||
}
|
||||
|
||||
async function expireClosedGrants(db: OutreachDb): Promise<void> {
|
||||
await serverExpireClosedGrants(db);
|
||||
}
|
||||
const expireClosedGrantsStep = DBOS.registerStep(expireClosedGrants, {
|
||||
name: 'expireClosedGrants',
|
||||
retriesAllowed: true,
|
||||
maxAttempts: 3,
|
||||
});
|
||||
|
||||
async function runExpireGrants(): Promise<void> {
|
||||
const { db } = getExpireGrantsDeps();
|
||||
await expireClosedGrantsStep(db);
|
||||
}
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachExpireGrantsRegistered?: boolean;
|
||||
};
|
||||
|
||||
if (!g.__outreachExpireGrantsRegistered) {
|
||||
g.__outreachExpireGrantsRegistered = true;
|
||||
|
||||
const expireGrants = async (_scheduledTime: Date, _startedAt: Date) => {
|
||||
try {
|
||||
await runExpireGrants();
|
||||
} catch (err) {
|
||||
console.error('[expire-grants] pass failed:', err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
DBOS.registerWorkflow(expireGrants, { name: 'expireGrants' });
|
||||
DBOS.registerScheduled(expireGrants, {
|
||||
crontab: '0 * * * *',
|
||||
name: 'expireGrants',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
137
apps/outreach-worker/src/workflows/ingest-grants.ts
Normal file
137
apps/outreach-worker/src/workflows/ingest-grants.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* Nightly grant ingestion workflow.
|
||||
*
|
||||
* Pulls open grant opportunities from upstream sources (currently just
|
||||
* Grants.gov; NH state postings and 990-PF extracts land as additional
|
||||
* fetch+normalize steps later) and upserts them into the outreach schema.
|
||||
*
|
||||
* Registration follows the pattern established in novelpad-desktop's
|
||||
* `packages/core/src/workflow/dbos/register-readiness-reconcile.server.ts`:
|
||||
* the scheduled function must ALSO be registered as a plain workflow (both
|
||||
* registrations referencing the same function object), and deps are pulled
|
||||
* from a module-scope registry populated before `DBOS.launch()` — DBOS
|
||||
* serializes workflow args, so closures/functions can't cross that boundary.
|
||||
*/
|
||||
import { DBOS, SchedulerMode } from '@dbos-inc/dbos-sdk';
|
||||
import type { schema } from '@novelpad/outreach-core';
|
||||
import {
|
||||
serverInsertGrants,
|
||||
type NewGrantInput,
|
||||
} from '@novelpad/outreach-core/server';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
export type OutreachDb = NodePgDatabase<typeof schema>;
|
||||
|
||||
/** Shape of a single opportunity as returned by the Grants.gov Search2 API. */
|
||||
export interface RawGrantsGovOpportunity {
|
||||
readonly opportunityId: string;
|
||||
readonly opportunityNumber: string;
|
||||
readonly opportunityTitle: string;
|
||||
readonly agencyCode: string | null;
|
||||
readonly openDate: string | null;
|
||||
readonly closeDate: string | null;
|
||||
readonly awardCeiling: number | null;
|
||||
readonly awardFloor: number | null;
|
||||
}
|
||||
|
||||
export interface IngestGrantsDeps {
|
||||
readonly db: OutreachDb;
|
||||
}
|
||||
|
||||
let registeredDeps: IngestGrantsDeps | null = null;
|
||||
|
||||
export function setIngestGrantsDeps(deps: IngestGrantsDeps): void {
|
||||
registeredDeps = deps;
|
||||
}
|
||||
|
||||
function getIngestGrantsDeps(): IngestGrantsDeps {
|
||||
if (registeredDeps == null) {
|
||||
throw new Error(
|
||||
'IngestGrantsDeps not registered. Call setIngestGrantsDeps() before DBOS.launch().',
|
||||
);
|
||||
}
|
||||
return registeredDeps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch open opportunities from Grants.gov.
|
||||
*
|
||||
* Real implementation calls the Grants.gov Search2 API
|
||||
* (`POST https://api.grants.gov/v1/api/search2`, JSON body with
|
||||
* `oppStatuses: "posted"` and pagination via `rows`/`startRecordNum`). Stubbed
|
||||
* to an empty array until that integration lands.
|
||||
*/
|
||||
async function fetchGrantsGov(): Promise<ReadonlyArray<RawGrantsGovOpportunity>> {
|
||||
// TODO(outreach): call Grants.gov Search2 API and paginate through results.
|
||||
return [];
|
||||
}
|
||||
const fetchGrantsGovStep = DBOS.registerStep(fetchGrantsGov, {
|
||||
name: 'fetchGrantsGov',
|
||||
retriesAllowed: true,
|
||||
maxAttempts: 3,
|
||||
});
|
||||
|
||||
function normalize(
|
||||
raw: ReadonlyArray<RawGrantsGovOpportunity>,
|
||||
): ReadonlyArray<NewGrantInput> {
|
||||
return raw.map((opp) => ({
|
||||
// Stable identity across re-crawls; also the schema's upsert key.
|
||||
sourceUrl: `https://www.grants.gov/search-results-detail/${opp.opportunityId}`,
|
||||
source: 'grants_gov' as const,
|
||||
title: opp.opportunityTitle,
|
||||
funder: opp.agencyCode ?? 'Unknown federal agency',
|
||||
openDate: opp.openDate == null ? null : new Date(opp.openDate),
|
||||
closeDate: opp.closeDate == null ? null : new Date(opp.closeDate),
|
||||
// Schema stores whole dollars (award amounts are never sub-dollar).
|
||||
awardFloor: opp.awardFloor == null ? null : Math.round(opp.awardFloor),
|
||||
awardCeiling: opp.awardCeiling == null ? null : Math.round(opp.awardCeiling),
|
||||
lastVerifiedAt: new Date(),
|
||||
}));
|
||||
}
|
||||
|
||||
async function upsertGrants(
|
||||
db: OutreachDb,
|
||||
grants: ReadonlyArray<NewGrantInput>,
|
||||
): Promise<void> {
|
||||
if (grants.length === 0) return;
|
||||
await serverInsertGrants(db, grants);
|
||||
}
|
||||
const upsertGrantsStep = DBOS.registerStep(upsertGrants, {
|
||||
name: 'upsertGrants',
|
||||
retriesAllowed: true,
|
||||
maxAttempts: 3,
|
||||
});
|
||||
|
||||
async function runIngestGrants(): Promise<void> {
|
||||
const { db } = getIngestGrantsDeps();
|
||||
|
||||
const raw = await fetchGrantsGovStep();
|
||||
const normalized = normalize(raw);
|
||||
await upsertGrantsStep(db, normalized);
|
||||
}
|
||||
|
||||
const g = globalThis as unknown as {
|
||||
__outreachIngestGrantsRegistered?: boolean;
|
||||
};
|
||||
|
||||
if (!g.__outreachIngestGrantsRegistered) {
|
||||
g.__outreachIngestGrantsRegistered = true;
|
||||
|
||||
const ingestGrants = async (_scheduledTime: Date, _startedAt: Date) => {
|
||||
try {
|
||||
await runIngestGrants();
|
||||
} catch (err) {
|
||||
console.error('[ingest-grants] pass failed:', err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// Must be registered as BOTH a workflow and a scheduled function,
|
||||
// referencing the same function object — see module doc comment.
|
||||
DBOS.registerWorkflow(ingestGrants, { name: 'ingestGrants' });
|
||||
DBOS.registerScheduled(ingestGrants, {
|
||||
crontab: '0 3 * * *',
|
||||
name: 'ingestGrants',
|
||||
mode: SchedulerMode.ExactlyOncePerInterval,
|
||||
});
|
||||
}
|
||||
18
apps/outreach-worker/tsconfig.json
Normal file
18
apps/outreach-worker/tsconfig.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"extends": "@novelpad/config/tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./build",
|
||||
"rootDir": "./src",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"target": "ES2022",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"],
|
||||
"noEmit": true,
|
||||
"paths": {
|
||||
"#~/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "build"]
|
||||
}
|
||||
Reference in New Issue
Block a user