Compare commits

..

5 Commits

Author SHA1 Message Date
Croissant Le Doux
b65ee4c8ad Memory layers: budgeted hot context (#27)
Reginald's context is tiered so the model always sees what matters without ever
copying ticket data into the prompt:
- HOT (this module) — charter + active directives + the focus snapshot, packed
  under a hard token budget (2k). Assembled fresh each turn as the prompt seed.
- WARM — the append-only directive/event ledger + digest, summarized on demand.
- COLD — gitea + sidecar via query_project. Ticket bodies/comments/detail live
  here and are NEVER inlined; the model fetches them by number when needed.

- `assembleHotContext(inputs, budget=2000)`: focus (tiny, always kept) → most
  recent active directives (each while they fit ~⅔) → charter fills the true
  remainder, truncated on a line boundary. Measures the fixed tail exactly and
  reserves for header/joiner/ellipsis so the total never exceeds budget.
- `estimateTokens` (tokenizer-free ~4 chars/token, slight over-estimate so a real
  tokenizer stays under), `activeDirectives` (accepted/amended, most-recent-first).

Acceptance met: hot assembles under the 2k budget even with a ~34k-token charter;
nothing ticket-shaped is inlined (only numbers + titles for focus). +5 core tests;
full core suite + typecheck green.

Follow-up: wire assembleHotContext into the live system prompt in main (needs
charter + directives + focus at chat time) — the assembly + budget is the tested core.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-09 15:37:57 -04:00
2a6413821a Merge pull request 'calibration: count same-day closes honestly (#34)' (#54) from feat/calibration-honesty into main
Reviewed-on: #54
2026-07-09 19:23:59 +00:00
008435f1c2 Merge branch 'main' into feat/calibration-honesty 2026-07-09 19:23:55 +00:00
354ba9227e Merge pull request 'apply_changes: unified mutation tool — estimate/priority/assign/milestone, in-app + agent (#24)' (#53) from feat/apply-changes-assign-milestone into main
Reviewed-on: #53
2026-07-09 19:23:49 +00:00
Croissant Le Doux
89c873b368 calibration: count same-day closes honestly (#34)
The cold-start surface showed "N/20 closed issues estimated", implying you're
just (20−N) closes away. But calibrationSamples silently drops closed+estimated
issues that closed in 0 working days (same-day closes) — real closes that
structurally can't calibrate. On this repo that's 10 of 24 closes hidden: the
note read 14/20 as if 6 away, when a third of the history will never count.

- core: `calibrationCoverage(issues, timelines, asOf)` → { candidates, usable,
  excludedSameDay }, counting the silently-excluded same-day closes. Pure, tested.
- surface it: CalibrationData gains `excludedSameDay`; backlogCalibration returns
  the coverage; the Runway note and the Calibration screen now say "… · N same-day
  closes can't calibrate" so the thin sample is explained, not just reported.

Verified on christian/commitea: closed=24, usable=14, excludedSameDay=10.
131 core green (incl. new coverage test); core + desktop typecheck; 14 fixture e2e.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-09 15:18:16 -04:00
10 changed files with 263 additions and 5 deletions

View File

@@ -178,6 +178,9 @@ export function CalibrationScreen({ onBack, data }: { onBack: () => void; data?:
{c.active
? 'You are not bad at estimating; you are optimistic in a very stable way. Stable, I can work with.'
: 'Not enough closed history yet — Im forecasting from cold-start priors and widening the cone to stay honest. The curve takes over at 20.'}
{!c.active && c.excludedSameDay > 0
? ` And ${c.excludedSameDay} closed ${c.excludedSameDay === 1 ? 'issue' : 'issues'} closed the same day they were started — 0 working days cant calibrate, so they dont count toward the 20.`
: ''}
</p>
</Card>
</div>

View File

@@ -17,7 +17,7 @@ export function RunwayScreen({
}: {
onOpenCalibration: () => void
onOpenMilestone: (id?: number) => void
calibration?: { n: number; coldStart: boolean }
calibration?: { n: number; coldStart: boolean; excludedSameDay?: number }
milestones?: RunwayMilestone[]
capacity?: CapacityMember[]
}) {
@@ -30,9 +30,11 @@ export function RunwayScreen({
hours: `${capacityPerWorkday(m).toFixed(2)} pd/day`,
}))
: CAPACITY
const excluded = calibration?.excludedSameDay ?? 0
const calibNote = calibration
? calibration.coldStart
? `cold-start priors · ${calibration.n}/20 closed issues estimated`
? `cold-start priors · ${calibration.n}/20 closed issues estimated` +
(excluded > 0 ? ` · ${excluded} same-day close${excluded === 1 ? '' : 's'} cant calibrate` : '')
: `calibrated on ${calibration.n} closed ${calibration.n === 1 ? 'issue' : 'issues'}`
: 'calibrated on 27 closed issues'
return (

View File

@@ -306,7 +306,15 @@ export function AppShell() {
setMilestoneId(id ?? null)
setView('milestone')
}}
calibration={calibration ? { n: calibration.model.n, coldStart: calibration.model.coldStart } : undefined}
calibration={
calibration
? {
n: calibration.model.n,
coldStart: calibration.model.coldStart,
excludedSameDay: calibration.coverage.excludedSameDay,
}
: undefined
}
milestones={runwayMilestones}
capacity={capacityMembers}
/>

View File

@@ -361,11 +361,14 @@ export interface CalibrationData {
scatter: number[][]
fit: number
effect: { raw: string; banded: string; p50: string }
/** Closed+estimated issues that can't calibrate (same-day / 0-day closes). */
excludedSameDay: number
}
export const CALIBRATION: CalibrationData = {
n: 27,
active: true,
excludedSameDay: 0,
labels: [
{ label: 'est/1d', n: 8, median: '1.1d', bias: 8 },
{ label: 'est/2d', n: 9, median: '2.4d', bias: 18 },

View File

@@ -1,6 +1,8 @@
import {
type CalibrationCoverage,
type CalibrationModel,
type CalibrationSample,
calibrationCoverage,
calibrationSamples,
type CapacityMember,
capacityPerWorkday,
@@ -171,10 +173,11 @@ export function backlogCalibration(
issues: GiteaIssue[],
timelines: Timelines = {},
asOf: Date = new Date(),
): { model: CalibrationModel; data: CalibrationData } {
): { model: CalibrationModel; data: CalibrationData; coverage: CalibrationCoverage } {
const samples = calibrationSamples(issues, timelines, asOf)
const model = fitCalibration(samples)
return { model, data: calibrationData(model, samples, issues) }
const coverage = calibrationCoverage(issues, timelines, asOf)
return { model, coverage, data: calibrationData(model, samples, issues, coverage.excludedSameDay) }
}
const pctFromMu = (mu: number) => Math.round((Math.exp(mu) - 1) * 100)
@@ -188,6 +191,7 @@ export function calibrationData(
model: CalibrationModel,
samples: CalibrationSample[],
openIssues: GiteaIssue[],
excludedSameDay = 0,
): CalibrationData {
const labels = PRIOR_BUCKETS.map((b) => {
const inBucket = samples.filter((s) => s.bucket === b)
@@ -227,6 +231,7 @@ export function calibrationData(
scatter: samples.map((s) => [s.estimateDays, s.actualWorkingDays]),
fit: Number(Math.exp(model.global.mu).toFixed(2)),
effect,
excludedSameDay,
}
}

View File

@@ -0,0 +1,71 @@
import { describe, expect, it } from 'vitest'
import type { DirectiveEntry } from '../directives/record-directive-v0.js'
import type { Focus, ScheduledItem } from '../scheduler/scheduler-v0.js'
import {
activeDirectives,
assembleHotContext,
estimateTokens,
HOT_CONTEXT_BUDGET_TOKENS,
} from './memory-v0.js'
const item = (number: number, title: string): ScheduledItem =>
({ number, title, labels: [], order: 0, startDay: 0, endDay: 1, durationDays: 1, blockedBy: [], blocks: [], critical: false, rationale: '' })
const focus: Focus = { now: item(7, 'Fix lifecycle inference'), next: item(8, 'Webhook listener'), later: null }
const directive = (over: Partial<DirectiveEntry>): DirectiveEntry => ({
id: 'd1',
ts: '2026-02-10T09:00:00Z',
status: 'accepted',
kind: 'note',
quote: 'pilots come first',
...over,
})
describe('memory-v0 (#27)', () => {
it('estimateTokens is a slight over-estimate (~4 chars/token)', () => {
expect(estimateTokens('')).toBe(0)
expect(estimateTokens('abcd')).toBe(1)
expect(estimateTokens('a'.repeat(4001))).toBe(1001)
})
it('activeDirectives keeps accepted/amended, most-recent-first', () => {
const ds = [
directive({ id: 'a', ts: '2026-02-01T00:00:00Z', status: 'accepted', quote: 'old' }),
directive({ id: 'b', ts: '2026-02-11T00:00:00Z', status: 'amended', quote: 'new' }),
directive({ id: 'c', ts: '2026-02-12T00:00:00Z', status: 'withdrawn', quote: 'gone' }),
directive({ id: 'd', ts: '2026-02-09T00:00:00Z', status: 'proposed', quote: 'maybe' }),
]
expect(activeDirectives(ds).map((d) => d.quote)).toEqual(['new', 'old'])
})
it('assembles hot context under the 2k budget even with a huge charter', () => {
const huge = 'Charter line that goes on and on. '.repeat(2000) // ~34k tokens
const ds = Array.from({ length: 50 }, (_, i) =>
directive({ id: `d${i}`, ts: `2026-02-${String((i % 27) + 1).padStart(2, '0')}T00:00:00Z`, quote: `directive number ${i}` }),
)
const out = assembleHotContext({ charter: huge, directives: ds, focus })
expect(estimateTokens(out)).toBeLessThanOrEqual(HOT_CONTEXT_BUDGET_TOKENS)
// focus (tiny) is always kept; the charter is the part that gets truncated
expect(out).toContain('## Focus')
expect(out).toContain('#7 Fix lifecycle inference')
expect(out).toContain('…') // charter was clamped
// at least some recent directives survived
expect(out).toContain('## Active directives')
})
it('never inlines ticket bodies — only numbers + titles appear for focus', () => {
// The assembler takes no issue bodies by construction; focus shows #number title only.
const out = assembleHotContext({ charter: 'Ship the beta.', directives: [directive({})], focus })
expect(out).toContain('Now: #7 Fix lifecycle inference')
expect(out).toContain('[note] pilots come first')
expect(out).not.toMatch(/body|description|comment/i)
})
it('degrades to just focus when there is no charter or directives', () => {
const out = assembleHotContext({ charter: '', directives: [], focus })
expect(out).toBe(['## Focus', 'Now: #7 Fix lifecycle inference', 'Next: #8 Webhook listener', 'Later: —'].join('\n'))
})
})

View File

@@ -0,0 +1,102 @@
/**
* Memory layers, v0 (#27). Reginald's context is tiered so the model always sees
* what matters without ever copying ticket data into the prompt:
*
* - HOT (this module) — charter + active directives + the focus snapshot, packed
* under a hard token budget. Assembled fresh each turn; it's the system-prompt seed.
* - WARM — the append-only directive/event ledger + periodic digest, in pm-state.
* Not inlined; summarized on demand.
* - COLD — gitea + the sidecar, reached through `query_project` tools. Ticket bodies,
* comments, and per-issue detail live here and are NEVER copied into memory —
* the model fetches them by number when it needs them.
*
* The invariant: HOT stays under budget, and nothing ticket-shaped is inlined.
*/
import type { DirectiveEntry } from '../directives/record-directive-v0.js'
import type { Focus } from '../scheduler/scheduler-v0.js'
/** The hot layer's hard ceiling (#27: hot context assembles under 2k tokens). */
export const HOT_CONTEXT_BUDGET_TOKENS = 2000
/**
* Tokenizer-free estimate (~4 chars/token). Deliberately a slight over-estimate so
* a real tokenizer never exceeds what this predicts — the budget stays safe.
*/
export function estimateTokens(text: string): number {
return Math.ceil(text.length / 4)
}
/** Directives that still bind: accepted or amended, most-recent-first. */
export function activeDirectives(all: DirectiveEntry[]): DirectiveEntry[] {
return all
.filter((d) => d.status === 'accepted' || d.status === 'amended')
.slice()
.sort((a, b) => (a.ts < b.ts ? 1 : a.ts > b.ts ? -1 : 0))
}
export interface HotContextInputs {
/** The project charter markdown (hot-memory seed). */
charter: string
/** The directive ledger (any status — filtered to active here). */
directives: DirectiveEntry[]
/** The current Now/Next/Later focus, or null when nothing is scheduled. */
focus: Focus | null
}
function focusBlock(focus: Focus | null): string {
if (!focus) return ''
const slot = (label: string, item: Focus['now']) => (item ? `${label}: #${item.number} ${item.title}` : `${label}: —`)
return ['## Focus', slot('Now', focus.now), slot('Next', focus.next), slot('Later', focus.later)].join('\n')
}
function directivesBlock(directives: DirectiveEntry[]): string[] {
// one compact line each; the verbatim quote is the payload, kind is the tag
return directives.map((d) => `- [${d.kind}] ${d.quote}`)
}
/** Truncate to a token budget on a whitespace boundary, with an ellipsis marker. */
function clampToTokens(text: string, budgetTokens: number): string {
if (estimateTokens(text) <= budgetTokens) return text
const maxChars = Math.max(0, budgetTokens * 4 - 1)
const cut = text.slice(0, maxChars)
const lastBreak = cut.lastIndexOf('\n')
return `${(lastBreak > maxChars * 0.6 ? cut.slice(0, lastBreak) : cut).trimEnd()}\n…`
}
/**
* Assemble the HOT context under `budget` tokens. Priority when space is tight:
* the focus snapshot (tiny, always kept) → the most recent active directives
* (each while they fit) → the charter fills whatever budget remains (truncated).
* Never inlines ticket bodies — only charter text, directive quotes, and focus
* titles, all authored/short. Returns a single prompt-ready block.
*/
export function assembleHotContext(inputs: HotContextInputs, budget = HOT_CONTEXT_BUDGET_TOKENS): string {
const focus = focusBlock(inputs.focus)
const focusTokens = focus ? estimateTokens(focus) : 0
// fit the most recent active directives into ~⅔ of what's left after focus
const active = activeDirectives(inputs.directives)
const directiveCap = Math.max(0, Math.floor((budget - focusTokens) * (2 / 3)))
const keptDirectives: string[] = []
let directiveTokens = 0
for (const line of directivesBlock(active)) {
const t = estimateTokens(line) + 1
if (directiveTokens + t > directiveCap) break
keptDirectives.push(line)
directiveTokens += t
}
const directives = keptDirectives.length ? ['## Active directives', ...keptDirectives].join('\n') : ''
// Measure the fixed tail (directives + focus, with their joiner) exactly, then
// give the charter the true remainder — reserving for the "## Charter" header,
// the block joiner, and the truncation ellipsis so the total never exceeds budget.
const tail = [directives, focus].filter(Boolean).join('\n\n')
const tailTokens = tail ? estimateTokens(tail) : 0
const reserve = estimateTokens(`## Charter\n${tail ? '\n\n' : ''}\n…`)
const charterBudget = Math.max(0, budget - tailTokens - reserve)
const charterBody = inputs.charter.trim() ? clampToTokens(inputs.charter.trim(), charterBudget) : ''
const charter = charterBody ? `## Charter\n${charterBody}` : ''
return [charter, tail].filter(Boolean).join('\n\n')
}

View File

@@ -5,6 +5,7 @@ import type { LifecycleEvent } from '../lifecycle/lifecycle-v0.js'
import type { GiteaIssue } from '../gitea/types.js'
import {
CALIBRATION_BUCKET_FLOOR,
calibrationCoverage,
calibrationSamples,
type CalibrationSample,
COLD_START_THRESHOLD,
@@ -118,4 +119,26 @@ describe('calibrationSamples', () => {
const noEst = issue({ number: 9, labels: [] })
expect(calibrationSamples([open, noEst], { ...events(8), ...events(9) }, asOf)).toEqual([])
})
it('coverage counts same-day closes as excluded candidates, not as "more closes needed"', () => {
// usable: commit Wed 01-07 → close Mon 01-12 = 3 working days
const usable = issue({ number: 7, labels: ['est/2d'] })
// same-day close: commit and close on the same day = 0 working days → excluded
const sameDay = issue({ number: 10, labels: ['est/2d'], createdAt: '2026-01-12T08:00:00Z' })
const sameDayEvents = {
10: [
{ type: 'commit', at: '2026-01-12T09:00:00Z' } as LifecycleEvent,
{ type: 'close', at: '2026-01-12T17:00:00Z' } as LifecycleEvent,
],
}
const open = issue({ number: 8, state: 'open', labels: ['est/2d'], closedAt: null })
const noEst = issue({ number: 9, labels: [] })
const cov = calibrationCoverage([usable, sameDay, open, noEst], { ...events(7), ...sameDayEvents }, asOf)
expect(cov.candidates).toBe(2) // closed + estimated only (usable + sameDay)
expect(cov.usable).toBe(1)
expect(cov.excludedSameDay).toBe(1)
// the honest denominator: usable matches the model's n
expect(cov.usable).toBe(calibrationSamples([usable, sameDay, open, noEst], { ...events(7), ...sameDayEvents }, asOf).length)
})
})

View File

@@ -125,3 +125,40 @@ export function calibrationSamples(
}
return out
}
/** How the closed+estimated backlog splits into usable samples vs. what can't calibrate. */
export interface CalibrationCoverage {
/** Closed issues carrying an estimate — the calibration candidates. */
candidates: number
/** Candidates that yielded a usable actual (> 0 working days) → become samples. */
usable: number
/**
* Candidates excluded because the issue closed with 0 working days (same-day
* close) or no resolvable actual — real closes that structurally can't
* calibrate. Counting them keeps `usable/threshold` honest: it's not "N more
* closes away" if some of your closes will never count.
*/
excludedSameDay: number
}
/**
* Coverage of the calibration candidates — how many closed+estimated issues are
* usable vs. silently unusable (same-day / 0-day closes). {@link calibrationSamples}
* drops the latter; this counts them so the UI can say *why* the sample is thin.
*/
export function calibrationCoverage(
issues: GiteaIssue[],
timelines: Record<number, LifecycleEvent[]>,
asOf: Date,
): CalibrationCoverage {
let candidates = 0
let usable = 0
for (const issue of issues) {
if (issue.state !== 'closed') continue
if (issue.facts.estimateDays == null) continue
candidates++
const inf = inferLifecycle(issue, timelines[issue.number] ?? [], asOf)
if (inf.actualWorkingDays != null && inf.actualWorkingDays > 0) usable++
}
return { candidates, usable, excludedSameDay: candidates - usable }
}

View File

@@ -80,6 +80,7 @@ export type {
export {
CALIBRATION_BUCKET_FLOOR,
calibrationCoverage,
calibrationSamples,
COLD_START_THRESHOLD,
fitCalibration,
@@ -87,6 +88,7 @@ export {
} from './calibration/calibration-v0.js'
export type {
BucketFit,
CalibrationCoverage,
CalibrationModel,
CalibrationSample,
PersonBias,
@@ -107,6 +109,8 @@ export {
} from './agent/agent-tools.js'
export { buildProjectView } from './agent/query-project.js'
export type { ProjectSnapshot, ProjectView, QueryFilters } from './agent/query-project.js'
export { activeDirectives, assembleHotContext, estimateTokens, HOT_CONTEXT_BUDGET_TOKENS } from './agent/memory-v0.js'
export type { HotContextInputs } from './agent/memory-v0.js'
export { CAPTURE_SYSTEM, captureWork, parseCaptureArgs, PROPOSE_ISSUES_TOOL } from './agent/capture-work.js'
export type { CaptureProposal, ProposedIssue } from './agent/capture-work.js'