feat: calibration from closed-issue actuals → forecast flips off cold-start (#1)
Close the D3 loop. The forecast now learns from the team's own estimate-vs-actual history (the working time #5 infers from git events) instead of guessing forever. core (@commitea/core/calibration-v0): - fitCalibration(samples): lognormal fit on log(actual/estimate) — global + per-bucket (once a bucket clears the floor) + per-person bias. coldStart until n >= 20 closed-with-estimate issues. - calibrationSamples(): pull those samples from the closed backlog via lifecycle inference (estimate label vs inferred actualWorkingDays). - toDurationModel(): project the fit to the params forecast consumes. - forecast() gains options.model: when past cold-start, fitted params drive the sim (per bucket, global fallback); otherwise the code priors do. Forecast.coldStart now reflects the model. nearestBucket extracted + exported. app: - AppShell fits calibration once from the reconciled backlog, feeds the model into forecastBacklog (cone), and drives the Calibration screen + Runway header. - Focus cone footer, Runway note, and Calibration screen now say cold-start (N/20) vs calibrated (on N closed) from real data; Calibration scatter / bucket bias / per-person all fitted, degrading honestly on a thin dataset. Known refinement: same-day closes yield 0 working-day actuals (day-granular) and are excluded, so a fast-moving repo can sit at n=0 — honest, but a fractional (hours-based) actual would let those count. Per-person uses gitea login, not display name, until the person map lands. Note: also re-lands #10 (Monte Carlo) and #5 (lifecycle) which merged into their stacked base branches but never propagated to main (stacked-merge trap); this branch is cut from main and carries all three so main is whole again. Verified: 74 core tests green (9 calibration + 2 forecast-switch added), desktop typecheck clean, 14 fixture e2e green, live spec asserts the real cold-start calibration surface (Runway note + screen badge fitted from actuals). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -97,4 +97,27 @@ describe('forecast', () => {
|
||||
expect(f.scope).toBe(2)
|
||||
expect(f.p50Day).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('a cold-start model changes nothing — the code priors still drive it', () => {
|
||||
const priors = forecast(scope, [], { trials: 1000, seed: 7 })
|
||||
const cold = forecast(scope, [], {
|
||||
trials: 1000,
|
||||
seed: 7,
|
||||
model: { coldStart: true, global: { mu: 5, sigma: 0.1 }, byBucket: {} },
|
||||
})
|
||||
expect(cold.coldStart).toBe(true)
|
||||
expect(cold.p50Day).toBeCloseTo(priors.p50Day, 6)
|
||||
})
|
||||
|
||||
it('a fitted model drives the sim once past cold-start', () => {
|
||||
// an optimistic fit (mu < 0, tight sigma) should land the scope sooner than the pessimistic priors
|
||||
const priors = forecast(scope, [], { trials: 2000, seed: 7 })
|
||||
const fitted = forecast(scope, [], {
|
||||
trials: 2000,
|
||||
seed: 7,
|
||||
model: { coldStart: false, global: { mu: -0.2, sigma: 0.1 }, byBucket: {} },
|
||||
})
|
||||
expect(fitted.coldStart).toBe(false)
|
||||
expect(fitted.p50Day).toBeLessThan(priors.p50Day)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -41,15 +41,29 @@ export const COLD_START_PRIORS: Record<number, LognormalPrior> = {
|
||||
8: { mu: 0.16, sigma: 0.36 },
|
||||
}
|
||||
|
||||
const PRIOR_BUCKETS = [1, 2, 3, 5, 8]
|
||||
export const PRIOR_BUCKETS = [1, 2, 3, 5, 8]
|
||||
|
||||
/** Nearest estimate bucket (ties resolve to the smaller bucket). */
|
||||
export function priorForEstimate(days: number): LognormalPrior {
|
||||
export function nearestBucket(days: number): number {
|
||||
let best = PRIOR_BUCKETS[0]
|
||||
for (const b of PRIOR_BUCKETS) {
|
||||
if (Math.abs(b - days) < Math.abs(best - days)) best = b
|
||||
}
|
||||
return COLD_START_PRIORS[best]
|
||||
return best
|
||||
}
|
||||
|
||||
/** The cold-start prior for the bucket nearest to `days`. */
|
||||
export function priorForEstimate(days: number): LognormalPrior {
|
||||
return COLD_START_PRIORS[nearestBucket(days)]
|
||||
}
|
||||
|
||||
/** The lognormal parameters `forecast` needs, per estimate bucket. */
|
||||
export interface DurationModel {
|
||||
coldStart: boolean
|
||||
/** Fallback params (used when a bucket lacks its own fit). */
|
||||
global: LognormalPrior
|
||||
/** Per-bucket fitted params; missing buckets fall back to `global`. */
|
||||
byBucket: Record<number, LognormalPrior>
|
||||
}
|
||||
|
||||
export interface ForecastOptions {
|
||||
@@ -57,6 +71,19 @@ export interface ForecastOptions {
|
||||
trials?: number
|
||||
/** PRNG seed. Fixed by default so a forecast is reproducible. */
|
||||
seed?: number
|
||||
/**
|
||||
* Fitted duration model. When present and not cold-start, its params drive
|
||||
* the sim; otherwise the code-resident cold-start priors do.
|
||||
*/
|
||||
model?: DurationModel
|
||||
}
|
||||
|
||||
/** Resolve the lognormal params for an estimate, preferring a fitted model. */
|
||||
export function durationParams(days: number, model?: DurationModel): LognormalPrior {
|
||||
if (model && !model.coldStart) {
|
||||
return model.byBucket[nearestBucket(days)] ?? model.global
|
||||
}
|
||||
return priorForEstimate(days)
|
||||
}
|
||||
|
||||
export interface BurnUpPoint {
|
||||
@@ -121,13 +148,14 @@ export function forecast(
|
||||
): Forecast {
|
||||
const trials = options.trials ?? DEFAULT_TRIALS
|
||||
const seed = options.seed ?? DEFAULT_SEED
|
||||
const coldStart = options.model ? options.model.coldStart : true
|
||||
const order = schedule(issues, edges).items // empty when a dependency cycle exists
|
||||
const n = order.length
|
||||
if (n === 0) {
|
||||
return { scope: 0, trials, coldStart: true, p50Day: 0, p80Day: 0, p95Day: 0, curve: [] }
|
||||
return { scope: 0, trials, coldStart, p50Day: 0, p80Day: 0, p95Day: 0, curve: [] }
|
||||
}
|
||||
|
||||
const priors = order.map((it) => priorForEstimate(it.durationDays))
|
||||
const priors = order.map((it) => durationParams(it.durationDays, options.model))
|
||||
const rng = mulberry32(seed)
|
||||
|
||||
// endByRank[k][t] = working day the (k+1)-th scheduled issue completes on trial t.
|
||||
@@ -156,7 +184,7 @@ export function forecast(
|
||||
return {
|
||||
scope: n,
|
||||
trials,
|
||||
coldStart: true,
|
||||
coldStart,
|
||||
p50Day: percentile(total, 0.5),
|
||||
p80Day: percentile(total, 0.8),
|
||||
p95Day: percentile(total, 0.95),
|
||||
|
||||
Reference in New Issue
Block a user