feat: calibration from closed-issue actuals → forecast flips off cold-start (#1)
Close the D3 loop. The forecast now learns from the team's own estimate-vs-actual history (the working time #5 infers from git events) instead of guessing forever. core (@commitea/core/calibration-v0): - fitCalibration(samples): lognormal fit on log(actual/estimate) — global + per-bucket (once a bucket clears the floor) + per-person bias. coldStart until n >= 20 closed-with-estimate issues. - calibrationSamples(): pull those samples from the closed backlog via lifecycle inference (estimate label vs inferred actualWorkingDays). - toDurationModel(): project the fit to the params forecast consumes. - forecast() gains options.model: when past cold-start, fitted params drive the sim (per bucket, global fallback); otherwise the code priors do. Forecast.coldStart now reflects the model. nearestBucket extracted + exported. app: - AppShell fits calibration once from the reconciled backlog, feeds the model into forecastBacklog (cone), and drives the Calibration screen + Runway header. - Focus cone footer, Runway note, and Calibration screen now say cold-start (N/20) vs calibrated (on N closed) from real data; Calibration scatter / bucket bias / per-person all fitted, degrading honestly on a thin dataset. Known refinement: same-day closes yield 0 working-day actuals (day-granular) and are excluded, so a fast-moving repo can sit at n=0 — honest, but a fractional (hours-based) actual would let those count. Per-person uses gitea login, not display name, until the person map lands. Note: also re-lands #10 (Monte Carlo) and #5 (lifecycle) which merged into their stacked base branches but never propagated to main (stacked-merge trap); this branch is cut from main and carries all three so main is whole again. Verified: 74 core tests green (9 calibration + 2 forecast-switch added), desktop typecheck clean, 14 fixture e2e green, live spec asserts the real cold-start calibration surface (Runway note + screen badge fitted from actuals). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -34,6 +34,17 @@ test.describe('live backlog', () => {
|
||||
await expect(win.getByText(/Cold-start priors/)).toBeVisible()
|
||||
await win.screenshot({ path: join(here, '.artifacts', 'screens', 'live-focus.png'), fullPage: true, animations: 'disabled' })
|
||||
|
||||
// Runway → calibration surface, fitted from real closed-issue actuals (#1).
|
||||
// With <20 estimated closes the repo is honestly cold-start; the note proves
|
||||
// the fit ran on real data, not the fixture's "calibrated on 27".
|
||||
await rail.getByRole('button', { name: 'Runway' }).click()
|
||||
await expect(
|
||||
win.getByText(/cold-start priors · \d+\/20 closed issues estimated|calibrated on \d+ closed/),
|
||||
).toBeVisible()
|
||||
await win.getByRole('button', { name: 'Full report' }).click()
|
||||
await expect(win.getByText(/cold-start · \d+\/20|curve active · n ≥ 20/)).toBeVisible()
|
||||
await win.screenshot({ path: join(here, '.artifacts', 'screens', 'live-calibration.png'), fullPage: true, animations: 'disabled' })
|
||||
|
||||
await app.close()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import React from 'react'
|
||||
|
||||
import { CALIBRATION } from '../../data/fixtures.js'
|
||||
import { CALIBRATION, type CalibrationData } from '../../data/fixtures.js'
|
||||
import { Badge, Card, Icon } from '../ui/index.js'
|
||||
|
||||
// Calibration report — estimate-vs-actual evidence behind the cones
|
||||
export function CalibrationScreen({ onBack }: { onBack: () => void }) {
|
||||
const c = CALIBRATION
|
||||
// Calibration report — estimate-vs-actual evidence behind the cones.
|
||||
// `data` (real fit from closed-issue actuals) overrides the demo fixture.
|
||||
export function CalibrationScreen({ onBack, data }: { onBack: () => void; data?: CalibrationData }) {
|
||||
const c = data ?? CALIBRATION
|
||||
|
||||
// scatter chart geometry
|
||||
const W = 420,
|
||||
@@ -68,7 +69,11 @@ export function CalibrationScreen({ onBack }: { onBack: () => void }) {
|
||||
<h1 style={{ font: 'var(--text-display)', color: 'var(--ink-1)', margin: 0 }}>Calibration</h1>
|
||||
<p style={{ font: 'var(--text-data)', color: 'var(--ink-3)', margin: '6px 0 0', whiteSpace: 'nowrap' }}>{c.n} closed issues with estimates · evidence, not opinion</p>
|
||||
</div>
|
||||
<Badge tone="ok" dot>curve active · n ≥ 20</Badge>
|
||||
{c.active ? (
|
||||
<Badge tone="ok" dot>curve active · n ≥ 20</Badge>
|
||||
) : (
|
||||
<Badge tone="warn" dot>cold-start · {c.n}/20</Badge>
|
||||
)}
|
||||
</header>
|
||||
</div>
|
||||
|
||||
@@ -170,7 +175,9 @@ export function CalibrationScreen({ onBack }: { onBack: () => void }) {
|
||||
<span style={{ font: '500 12.5px var(--font-mono)', color: 'var(--ink-1)', whiteSpace: 'nowrap' }}>{c.effect.banded}</span>
|
||||
</div>
|
||||
<p style={{ font: 'var(--text-agent)', color: 'var(--ink-2)', margin: '10px 0 0' }}>
|
||||
You are not bad at estimating; you are optimistic in a very stable way. Stable, I can work with.
|
||||
{c.active
|
||||
? 'You are not bad at estimating; you are optimistic in a very stable way. Stable, I can work with.'
|
||||
: 'Not enough closed history yet — I’m forecasting from cold-start priors and widening the cone to stay honest. The curve takes over at 20.'}
|
||||
</p>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
@@ -109,7 +109,9 @@ export function FocusScreen({
|
||||
<BurnUpCone data={forecast?.cone} />
|
||||
<p style={{ font: 'var(--text-agent)', color: 'var(--ink-2)', margin: '10px 0 0' }}>
|
||||
{forecast
|
||||
? `${forecast.scope} open ${forecast.scope === 1 ? 'issue' : 'issues'} in scope. Cold-start priors — the cone tightens as the team closes work.`
|
||||
? forecast.coldStart
|
||||
? `${forecast.scope} open ${forecast.scope === 1 ? 'issue' : 'issues'} in scope. Cold-start priors — ${forecast.calibratedN}/20 estimated closes so far; the cone tightens as the team closes work.`
|
||||
: `${forecast.scope} open ${forecast.scope === 1 ? 'issue' : 'issues'} in scope, calibrated on ${forecast.calibratedN} closed ${forecast.calibratedN === 1 ? 'issue' : 'issues'} of your own.`
|
||||
: 'The cone has narrowed since Friday. I’m quietly pleased.'}
|
||||
</p>
|
||||
</Card>
|
||||
|
||||
@@ -8,15 +8,22 @@ import { RUNWAY, CAPACITY } from '../../data/fixtures.js'
|
||||
export function RunwayScreen({
|
||||
onOpenCalibration,
|
||||
onOpenMilestone,
|
||||
calibration,
|
||||
}: {
|
||||
onOpenCalibration: () => void
|
||||
onOpenMilestone: () => void
|
||||
calibration?: { n: number; coldStart: boolean }
|
||||
}) {
|
||||
const calibNote = calibration
|
||||
? calibration.coldStart
|
||||
? `cold-start priors · ${calibration.n}/20 closed issues estimated`
|
||||
: `calibrated on ${calibration.n} closed ${calibration.n === 1 ? 'issue' : 'issues'}`
|
||||
: 'calibrated on 27 closed issues'
|
||||
return (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
|
||||
<header style={{ borderBottom: 'var(--rule-double)', paddingBottom: 14 }}>
|
||||
<h1 style={{ font: 'var(--text-display)', color: 'var(--ink-1)', margin: 0 }}>Runway</h1>
|
||||
<p style={{ font: 'var(--text-data)', color: 'var(--ink-3)', margin: '6px 0 0' }}>capacity vs milestone dates · calibrated on 27 closed issues</p>
|
||||
<p style={{ font: 'var(--text-data)', color: 'var(--ink-3)', margin: '6px 0 0' }}>capacity vs milestone dates · {calibNote}</p>
|
||||
</header>
|
||||
|
||||
<Card overline="Milestones" flush>
|
||||
|
||||
@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react'
|
||||
|
||||
import logoIcon from '../../design/assets/logo-icon.png'
|
||||
import type { IssueRef } from '../../data/fixtures.js'
|
||||
import { forecastBacklog, issuesToBoardColumns, scheduleFocus } from '../../lib/backlog.js'
|
||||
import { backlogCalibration, forecastBacklog, issuesToBoardColumns, scheduleFocus } from '../../lib/backlog.js'
|
||||
import { useBacklog } from '../../lib/use-backlog.js'
|
||||
import { PrimitivesGallery } from '../gallery.js'
|
||||
import { BoardScreen } from '../screens/board-screen.js'
|
||||
@@ -90,8 +90,12 @@ export function AppShell() {
|
||||
backlog.status === 'ready' ? issuesToBoardColumns(backlog.issues, backlog.timelines) : undefined
|
||||
const focus =
|
||||
backlog.status === 'ready' ? scheduleFocus(backlog.issues, backlog.deps, backlog.timelines) : undefined
|
||||
const calibration =
|
||||
backlog.status === 'ready' ? backlogCalibration(backlog.issues, backlog.timelines) : undefined
|
||||
const forecast =
|
||||
backlog.status === 'ready' ? (forecastBacklog(backlog.issues, backlog.deps) ?? undefined) : undefined
|
||||
backlog.status === 'ready'
|
||||
? (forecastBacklog(backlog.issues, backlog.deps, new Date(), calibration?.model) ?? undefined)
|
||||
: undefined
|
||||
|
||||
useEffect(() => {
|
||||
document.documentElement.setAttribute('data-theme', dark ? 'dark' : 'light')
|
||||
@@ -178,10 +182,11 @@ export function AppShell() {
|
||||
<RunwayScreen
|
||||
onOpenCalibration={() => setView('calibration')}
|
||||
onOpenMilestone={() => setView('milestone')}
|
||||
calibration={calibration ? { n: calibration.model.n, coldStart: calibration.model.coldStart } : undefined}
|
||||
/>
|
||||
)
|
||||
case 'calibration':
|
||||
return <CalibrationScreen onBack={() => setView('runway')} />
|
||||
return <CalibrationScreen onBack={() => setView('runway')} data={calibration?.data} />
|
||||
case 'milestone':
|
||||
return <MilestoneScreen onBack={() => setView('runway')} onOpenIssue={openIssue} />
|
||||
case 'inbox':
|
||||
|
||||
@@ -1,17 +1,24 @@
|
||||
import {
|
||||
type CalibrationModel,
|
||||
type CalibrationSample,
|
||||
calibrationSamples,
|
||||
COLD_START_THRESHOLD,
|
||||
type DependencyEdge,
|
||||
fitCalibration,
|
||||
forecast,
|
||||
type GiteaIssue,
|
||||
inferLifecycle,
|
||||
type LifecycleColumn,
|
||||
type LifecycleEvent,
|
||||
type LifecycleInference,
|
||||
PRIOR_BUCKETS,
|
||||
schedule,
|
||||
type ScheduledItem,
|
||||
selectFocus,
|
||||
toDurationModel,
|
||||
} from '@commitea/core'
|
||||
|
||||
import { type BoardColumn, type BoardIssue, type FocusIssue } from '../data/fixtures.js'
|
||||
import { type BoardColumn, type BoardIssue, type CalibrationData, type FocusIssue } from '../data/fixtures.js'
|
||||
import { type BurnUpData, buildBurnUpData } from './dates.js'
|
||||
|
||||
type Timelines = Record<number, LifecycleEvent[]>
|
||||
@@ -100,22 +107,109 @@ export interface ForecastView {
|
||||
cone: BurnUpData
|
||||
p80Label: string
|
||||
rangeLabel: string
|
||||
/** true while the forecast still runs on code priors (calibration not yet trusted). */
|
||||
coldStart: boolean
|
||||
/** Closed-with-estimate issues feeding calibration so far. */
|
||||
calibratedN: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Monte Carlo forecast over the open backlog, mapped onto a calendar-anchored
|
||||
* burn-up cone. Returns null when there's nothing to forecast (no open scope) —
|
||||
* the UI then falls back to the demo cone. `today` is injectable for tests.
|
||||
* burn-up cone. When a calibration model is supplied and past cold-start, its
|
||||
* fitted params drive the sim. Returns null when there's nothing to forecast.
|
||||
* `today` is injectable for tests.
|
||||
*/
|
||||
export function forecastBacklog(
|
||||
issues: GiteaIssue[],
|
||||
deps: DependencyEdge[],
|
||||
today: Date = new Date(),
|
||||
calibration?: CalibrationModel,
|
||||
): ForecastView | null {
|
||||
const f = forecast(toSchedulable(issues), deps)
|
||||
const model = calibration ? toDurationModel(calibration) : undefined
|
||||
const f = forecast(toSchedulable(issues), deps, model ? { model } : {})
|
||||
const cone = buildBurnUpData(f, today)
|
||||
if (!cone) return null
|
||||
return { scope: f.scope, cone, p80Label: cone.p80Label, rangeLabel: cone.rangeLabel }
|
||||
return {
|
||||
scope: f.scope,
|
||||
cone,
|
||||
p80Label: cone.p80Label,
|
||||
rangeLabel: cone.rangeLabel,
|
||||
coldStart: f.coldStart,
|
||||
calibratedN: calibration?.n ?? 0,
|
||||
}
|
||||
}
|
||||
|
||||
/** Fit the calibration model from the closed backlog's inferred actuals (#1). */
|
||||
export function calibrateBacklog(
|
||||
issues: GiteaIssue[],
|
||||
timelines: Timelines = {},
|
||||
asOf: Date = new Date(),
|
||||
): CalibrationModel {
|
||||
return fitCalibration(calibrationSamples(issues, timelines, asOf))
|
||||
}
|
||||
|
||||
/** Calibration model + its screen view in one pass over the closed backlog. */
|
||||
export function backlogCalibration(
|
||||
issues: GiteaIssue[],
|
||||
timelines: Timelines = {},
|
||||
asOf: Date = new Date(),
|
||||
): { model: CalibrationModel; data: CalibrationData } {
|
||||
const samples = calibrationSamples(issues, timelines, asOf)
|
||||
const model = fitCalibration(samples)
|
||||
return { model, data: calibrationData(model, samples, issues) }
|
||||
}
|
||||
|
||||
const pctFromMu = (mu: number) => Math.round((Math.exp(mu) - 1) * 100)
|
||||
|
||||
/**
|
||||
* Shape the calibration model + its samples into the screen's view. Buckets and
|
||||
* people only earn a bias once their sample clears the fit floor; everything
|
||||
* degrades honestly on a thin (cold-start) dataset.
|
||||
*/
|
||||
export function calibrationData(
|
||||
model: CalibrationModel,
|
||||
samples: CalibrationSample[],
|
||||
openIssues: GiteaIssue[],
|
||||
): CalibrationData {
|
||||
const labels = PRIOR_BUCKETS.map((b) => {
|
||||
const inBucket = samples.filter((s) => s.bucket === b)
|
||||
const fit = model.byBucket[b]
|
||||
const mu = fit ? fit.mu : model.global.mu
|
||||
return {
|
||||
label: `est/${b}d`,
|
||||
n: fit ? fit.n : inBucket.length,
|
||||
median: inBucket.length ? `${(b * Math.exp(mu)).toFixed(1)}d` : '—',
|
||||
bias: fit ? pctFromMu(fit.mu) : null,
|
||||
}
|
||||
})
|
||||
|
||||
const people = Object.entries(model.byPerson).map(([who, pb]) => ({
|
||||
who,
|
||||
n: pb.n,
|
||||
bias: pctFromMu(model.global.mu + pb.biasMu),
|
||||
note: '',
|
||||
}))
|
||||
|
||||
const openEst = openIssues
|
||||
.filter((i) => i.state === 'open')
|
||||
.reduce((sum, i) => sum + (i.facts.estimateDays ?? 2), 0)
|
||||
const effect = model.coldStart
|
||||
? { raw: `${model.n}/${COLD_START_THRESHOLD} estimated closes`, banded: 'cold-start priors', p50: '—' }
|
||||
: {
|
||||
raw: `${openEst}d estimated`,
|
||||
banded: `×${Math.exp(model.global.mu).toFixed(2)} median drift`,
|
||||
p50: `≈${Math.round(openEst * Math.exp(model.global.mu))}d`,
|
||||
}
|
||||
|
||||
return {
|
||||
n: model.n,
|
||||
active: !model.coldStart,
|
||||
labels,
|
||||
people,
|
||||
scatter: samples.map((s) => [s.estimateDays, s.actualWorkingDays]),
|
||||
fit: Number(Math.exp(model.global.mu).toFixed(2)),
|
||||
effect,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user