feat: calibration from closed-issue actuals → forecast flips off cold-start (#1)

Close the D3 loop. The forecast now learns from the team's own estimate-vs-actual
history (the working time #5 infers from git events) instead of guessing forever.

core (@commitea/core/calibration-v0):
- fitCalibration(samples): lognormal fit on log(actual/estimate) — global +
  per-bucket (once a bucket clears the floor) + per-person bias. coldStart until
  n >= 20 closed-with-estimate issues.
- calibrationSamples(): pull those samples from the closed backlog via lifecycle
  inference (estimate label vs inferred actualWorkingDays).
- toDurationModel(): project the fit to the params forecast consumes.
- forecast() gains options.model: when past cold-start, fitted params drive the
  sim (per bucket, global fallback); otherwise the code priors do. Forecast.coldStart
  now reflects the model. nearestBucket extracted + exported.

app:
- AppShell fits calibration once from the reconciled backlog, feeds the model into
  forecastBacklog (cone), and drives the Calibration screen + Runway header.
- Focus cone footer, Runway note, and Calibration screen now say cold-start (N/20)
  vs calibrated (on N closed) from real data; Calibration scatter / bucket bias /
  per-person all fitted, degrading honestly on a thin dataset.

Known refinement: same-day closes yield 0 working-day actuals (day-granular) and
are excluded, so a fast-moving repo can sit at n=0 — honest, but a fractional
(hours-based) actual would let those count. Per-person uses gitea login, not
display name, until the person map lands.

Note: also re-lands #10 (Monte Carlo) and #5 (lifecycle) which merged into their
stacked base branches but never propagated to main (stacked-merge trap); this
branch is cut from main and carries all three so main is whole again.

Verified: 74 core tests green (9 calibration + 2 forecast-switch added), desktop
typecheck clean, 14 fixture e2e green, live spec asserts the real cold-start
calibration surface (Runway note + screen badge fitted from actuals).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
2026-07-08 19:51:50 -04:00
parent 9cedd8646e
commit 7e26de1b6c
11 changed files with 476 additions and 24 deletions

View File

@@ -1,11 +1,12 @@
import React from 'react'
import { CALIBRATION } from '../../data/fixtures.js'
import { CALIBRATION, type CalibrationData } from '../../data/fixtures.js'
import { Badge, Card, Icon } from '../ui/index.js'
// Calibration report — estimate-vs-actual evidence behind the cones
export function CalibrationScreen({ onBack }: { onBack: () => void }) {
const c = CALIBRATION
// Calibration report — estimate-vs-actual evidence behind the cones.
// `data` (real fit from closed-issue actuals) overrides the demo fixture.
export function CalibrationScreen({ onBack, data }: { onBack: () => void; data?: CalibrationData }) {
const c = data ?? CALIBRATION
// scatter chart geometry
const W = 420,
@@ -68,7 +69,11 @@ export function CalibrationScreen({ onBack }: { onBack: () => void }) {
<h1 style={{ font: 'var(--text-display)', color: 'var(--ink-1)', margin: 0 }}>Calibration</h1>
<p style={{ font: 'var(--text-data)', color: 'var(--ink-3)', margin: '6px 0 0', whiteSpace: 'nowrap' }}>{c.n} closed issues with estimates · evidence, not opinion</p>
</div>
<Badge tone="ok" dot>curve active · n 20</Badge>
{c.active ? (
<Badge tone="ok" dot>curve active · n 20</Badge>
) : (
<Badge tone="warn" dot>cold-start · {c.n}/20</Badge>
)}
</header>
</div>
@@ -170,7 +175,9 @@ export function CalibrationScreen({ onBack }: { onBack: () => void }) {
<span style={{ font: '500 12.5px var(--font-mono)', color: 'var(--ink-1)', whiteSpace: 'nowrap' }}>{c.effect.banded}</span>
</div>
<p style={{ font: 'var(--text-agent)', color: 'var(--ink-2)', margin: '10px 0 0' }}>
You are not bad at estimating; you are optimistic in a very stable way. Stable, I can work with.
{c.active
? 'You are not bad at estimating; you are optimistic in a very stable way. Stable, I can work with.'
: 'Not enough closed history yet — Im forecasting from cold-start priors and widening the cone to stay honest. The curve takes over at 20.'}
</p>
</Card>
</div>

View File

@@ -109,7 +109,9 @@ export function FocusScreen({
<BurnUpCone data={forecast?.cone} />
<p style={{ font: 'var(--text-agent)', color: 'var(--ink-2)', margin: '10px 0 0' }}>
{forecast
? `${forecast.scope} open ${forecast.scope === 1 ? 'issue' : 'issues'} in scope. Cold-start priors — the cone tightens as the team closes work.`
? forecast.coldStart
? `${forecast.scope} open ${forecast.scope === 1 ? 'issue' : 'issues'} in scope. Cold-start priors — ${forecast.calibratedN}/20 estimated closes so far; the cone tightens as the team closes work.`
: `${forecast.scope} open ${forecast.scope === 1 ? 'issue' : 'issues'} in scope, calibrated on ${forecast.calibratedN} closed ${forecast.calibratedN === 1 ? 'issue' : 'issues'} of your own.`
: 'The cone has narrowed since Friday. Im quietly pleased.'}
</p>
</Card>

View File

@@ -8,15 +8,22 @@ import { RUNWAY, CAPACITY } from '../../data/fixtures.js'
export function RunwayScreen({
onOpenCalibration,
onOpenMilestone,
calibration,
}: {
onOpenCalibration: () => void
onOpenMilestone: () => void
calibration?: { n: number; coldStart: boolean }
}) {
const calibNote = calibration
? calibration.coldStart
? `cold-start priors · ${calibration.n}/20 closed issues estimated`
: `calibrated on ${calibration.n} closed ${calibration.n === 1 ? 'issue' : 'issues'}`
: 'calibrated on 27 closed issues'
return (
<div style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
<header style={{ borderBottom: 'var(--rule-double)', paddingBottom: 14 }}>
<h1 style={{ font: 'var(--text-display)', color: 'var(--ink-1)', margin: 0 }}>Runway</h1>
<p style={{ font: 'var(--text-data)', color: 'var(--ink-3)', margin: '6px 0 0' }}>capacity vs milestone dates · calibrated on 27 closed issues</p>
<p style={{ font: 'var(--text-data)', color: 'var(--ink-3)', margin: '6px 0 0' }}>capacity vs milestone dates · {calibNote}</p>
</header>
<Card overline="Milestones" flush>

View File

@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react'
import logoIcon from '../../design/assets/logo-icon.png'
import type { IssueRef } from '../../data/fixtures.js'
import { forecastBacklog, issuesToBoardColumns, scheduleFocus } from '../../lib/backlog.js'
import { backlogCalibration, forecastBacklog, issuesToBoardColumns, scheduleFocus } from '../../lib/backlog.js'
import { useBacklog } from '../../lib/use-backlog.js'
import { PrimitivesGallery } from '../gallery.js'
import { BoardScreen } from '../screens/board-screen.js'
@@ -90,8 +90,12 @@ export function AppShell() {
backlog.status === 'ready' ? issuesToBoardColumns(backlog.issues, backlog.timelines) : undefined
const focus =
backlog.status === 'ready' ? scheduleFocus(backlog.issues, backlog.deps, backlog.timelines) : undefined
const calibration =
backlog.status === 'ready' ? backlogCalibration(backlog.issues, backlog.timelines) : undefined
const forecast =
backlog.status === 'ready' ? (forecastBacklog(backlog.issues, backlog.deps) ?? undefined) : undefined
backlog.status === 'ready'
? (forecastBacklog(backlog.issues, backlog.deps, new Date(), calibration?.model) ?? undefined)
: undefined
useEffect(() => {
document.documentElement.setAttribute('data-theme', dark ? 'dark' : 'light')
@@ -178,10 +182,11 @@ export function AppShell() {
<RunwayScreen
onOpenCalibration={() => setView('calibration')}
onOpenMilestone={() => setView('milestone')}
calibration={calibration ? { n: calibration.model.n, coldStart: calibration.model.coldStart } : undefined}
/>
)
case 'calibration':
return <CalibrationScreen onBack={() => setView('runway')} />
return <CalibrationScreen onBack={() => setView('runway')} data={calibration?.data} />
case 'milestone':
return <MilestoneScreen onBack={() => setView('runway')} onOpenIssue={openIssue} />
case 'inbox':

View File

@@ -1,17 +1,24 @@
import {
type CalibrationModel,
type CalibrationSample,
calibrationSamples,
COLD_START_THRESHOLD,
type DependencyEdge,
fitCalibration,
forecast,
type GiteaIssue,
inferLifecycle,
type LifecycleColumn,
type LifecycleEvent,
type LifecycleInference,
PRIOR_BUCKETS,
schedule,
type ScheduledItem,
selectFocus,
toDurationModel,
} from '@commitea/core'
import { type BoardColumn, type BoardIssue, type FocusIssue } from '../data/fixtures.js'
import { type BoardColumn, type BoardIssue, type CalibrationData, type FocusIssue } from '../data/fixtures.js'
import { type BurnUpData, buildBurnUpData } from './dates.js'
type Timelines = Record<number, LifecycleEvent[]>
@@ -100,22 +107,109 @@ export interface ForecastView {
cone: BurnUpData
p80Label: string
rangeLabel: string
/** true while the forecast still runs on code priors (calibration not yet trusted). */
coldStart: boolean
/** Closed-with-estimate issues feeding calibration so far. */
calibratedN: number
}
/**
* Monte Carlo forecast over the open backlog, mapped onto a calendar-anchored
* burn-up cone. Returns null when there's nothing to forecast (no open scope) —
* the UI then falls back to the demo cone. `today` is injectable for tests.
* burn-up cone. When a calibration model is supplied and past cold-start, its
* fitted params drive the sim. Returns null when there's nothing to forecast.
* `today` is injectable for tests.
*/
export function forecastBacklog(
issues: GiteaIssue[],
deps: DependencyEdge[],
today: Date = new Date(),
calibration?: CalibrationModel,
): ForecastView | null {
const f = forecast(toSchedulable(issues), deps)
const model = calibration ? toDurationModel(calibration) : undefined
const f = forecast(toSchedulable(issues), deps, model ? { model } : {})
const cone = buildBurnUpData(f, today)
if (!cone) return null
return { scope: f.scope, cone, p80Label: cone.p80Label, rangeLabel: cone.rangeLabel }
return {
scope: f.scope,
cone,
p80Label: cone.p80Label,
rangeLabel: cone.rangeLabel,
coldStart: f.coldStart,
calibratedN: calibration?.n ?? 0,
}
}
/** Fit the calibration model from the closed backlog's inferred actuals (#1). */
export function calibrateBacklog(
issues: GiteaIssue[],
timelines: Timelines = {},
asOf: Date = new Date(),
): CalibrationModel {
return fitCalibration(calibrationSamples(issues, timelines, asOf))
}
/** Calibration model + its screen view in one pass over the closed backlog. */
export function backlogCalibration(
issues: GiteaIssue[],
timelines: Timelines = {},
asOf: Date = new Date(),
): { model: CalibrationModel; data: CalibrationData } {
const samples = calibrationSamples(issues, timelines, asOf)
const model = fitCalibration(samples)
return { model, data: calibrationData(model, samples, issues) }
}
const pctFromMu = (mu: number) => Math.round((Math.exp(mu) - 1) * 100)
/**
* Shape the calibration model + its samples into the screen's view. Buckets and
* people only earn a bias once their sample clears the fit floor; everything
* degrades honestly on a thin (cold-start) dataset.
*/
export function calibrationData(
model: CalibrationModel,
samples: CalibrationSample[],
openIssues: GiteaIssue[],
): CalibrationData {
const labels = PRIOR_BUCKETS.map((b) => {
const inBucket = samples.filter((s) => s.bucket === b)
const fit = model.byBucket[b]
const mu = fit ? fit.mu : model.global.mu
return {
label: `est/${b}d`,
n: fit ? fit.n : inBucket.length,
median: inBucket.length ? `${(b * Math.exp(mu)).toFixed(1)}d` : '—',
bias: fit ? pctFromMu(fit.mu) : null,
}
})
const people = Object.entries(model.byPerson).map(([who, pb]) => ({
who,
n: pb.n,
bias: pctFromMu(model.global.mu + pb.biasMu),
note: '',
}))
const openEst = openIssues
.filter((i) => i.state === 'open')
.reduce((sum, i) => sum + (i.facts.estimateDays ?? 2), 0)
const effect = model.coldStart
? { raw: `${model.n}/${COLD_START_THRESHOLD} estimated closes`, banded: 'cold-start priors', p50: '—' }
: {
raw: `${openEst}d estimated`,
banded: `×${Math.exp(model.global.mu).toFixed(2)} median drift`,
p50: `${Math.round(openEst * Math.exp(model.global.mu))}d`,
}
return {
n: model.n,
active: !model.coldStart,
labels,
people,
scatter: samples.map((s) => [s.estimateDays, s.actualWorkingDays]),
fit: Number(Math.exp(model.global.mu).toFixed(2)),
effect,
}
}
/**