From 89c873b3682583a5896fb36e18dff104a925ff4b Mon Sep 17 00:00:00 2001 From: Croissant Le Doux Date: Thu, 9 Jul 2026 15:18:16 -0400 Subject: [PATCH] calibration: count same-day closes honestly (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cold-start surface showed "N/20 closed issues estimated", implying you're just (20−N) closes away. But calibrationSamples silently drops closed+estimated issues that closed in 0 working days (same-day closes) — real closes that structurally can't calibrate. On this repo that's 10 of 24 closes hidden: the note read 14/20 as if 6 away, when a third of the history will never count. - core: `calibrationCoverage(issues, timelines, asOf)` → { candidates, usable, excludedSameDay }, counting the silently-excluded same-day closes. Pure, tested. - surface it: CalibrationData gains `excludedSameDay`; backlogCalibration returns the coverage; the Runway note and the Calibration screen now say "… · N same-day closes can't calibrate" so the thin sample is explained, not just reported. Verified on christian/commitea: closed=24, usable=14, excludedSameDay=10. 131 core green (incl. new coverage test); core + desktop typecheck; 14 fixture e2e. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../components/screens/calibration-screen.tsx | 3 ++ .../src/components/screens/runway-screen.tsx | 6 ++- .../src/components/shell/app-shell.tsx | 10 ++++- .../desktop/src/renderer/src/data/fixtures.ts | 3 ++ apps/desktop/src/renderer/src/lib/backlog.ts | 9 ++++- .../src/calibration/calibration-v0.test.ts | 23 ++++++++++++ .../core/src/calibration/calibration-v0.ts | 37 +++++++++++++++++++ packages/core/src/index.ts | 2 + 8 files changed, 88 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/renderer/src/components/screens/calibration-screen.tsx b/apps/desktop/src/renderer/src/components/screens/calibration-screen.tsx index f11a2b4..c728eda 100644 --- a/apps/desktop/src/renderer/src/components/screens/calibration-screen.tsx +++ b/apps/desktop/src/renderer/src/components/screens/calibration-screen.tsx @@ -178,6 +178,9 @@ export function CalibrationScreen({ onBack, data }: { onBack: () => void; data?: {c.active ? 'You are not bad at estimating; you are optimistic in a very stable way. Stable, I can work with.' : 'Not enough closed history yet — I’m forecasting from cold-start priors and widening the cone to stay honest. The curve takes over at 20.'} + {!c.active && c.excludedSameDay > 0 + ? ` And ${c.excludedSameDay} closed ${c.excludedSameDay === 1 ? 'issue' : 'issues'} closed the same day they were started — 0 working days can’t calibrate, so they don’t count toward the 20.` + : ''}

diff --git a/apps/desktop/src/renderer/src/components/screens/runway-screen.tsx b/apps/desktop/src/renderer/src/components/screens/runway-screen.tsx index e929110..8770ccb 100644 --- a/apps/desktop/src/renderer/src/components/screens/runway-screen.tsx +++ b/apps/desktop/src/renderer/src/components/screens/runway-screen.tsx @@ -17,7 +17,7 @@ export function RunwayScreen({ }: { onOpenCalibration: () => void onOpenMilestone: (id?: number) => void - calibration?: { n: number; coldStart: boolean } + calibration?: { n: number; coldStart: boolean; excludedSameDay?: number } milestones?: RunwayMilestone[] capacity?: CapacityMember[] }) { @@ -30,9 +30,11 @@ export function RunwayScreen({ hours: `${capacityPerWorkday(m).toFixed(2)} pd/day`, })) : CAPACITY + const excluded = calibration?.excludedSameDay ?? 0 const calibNote = calibration ? calibration.coldStart - ? `cold-start priors · ${calibration.n}/20 closed issues estimated` + ? `cold-start priors · ${calibration.n}/20 closed issues estimated` + + (excluded > 0 ? ` · ${excluded} same-day close${excluded === 1 ? '' : 's'} can’t calibrate` : '') : `calibrated on ${calibration.n} closed ${calibration.n === 1 ? 'issue' : 'issues'}` : 'calibrated on 27 closed issues' return ( diff --git a/apps/desktop/src/renderer/src/components/shell/app-shell.tsx b/apps/desktop/src/renderer/src/components/shell/app-shell.tsx index c7e8187..ff934c9 100644 --- a/apps/desktop/src/renderer/src/components/shell/app-shell.tsx +++ b/apps/desktop/src/renderer/src/components/shell/app-shell.tsx @@ -241,7 +241,15 @@ export function AppShell() { setMilestoneId(id ?? null) setView('milestone') }} - calibration={calibration ? { n: calibration.model.n, coldStart: calibration.model.coldStart } : undefined} + calibration={ + calibration + ? { + n: calibration.model.n, + coldStart: calibration.model.coldStart, + excludedSameDay: calibration.coverage.excludedSameDay, + } + : undefined + } milestones={runwayMilestones} capacity={capacityMembers} /> diff --git a/apps/desktop/src/renderer/src/data/fixtures.ts b/apps/desktop/src/renderer/src/data/fixtures.ts index ccc1a02..61b5190 100644 --- a/apps/desktop/src/renderer/src/data/fixtures.ts +++ b/apps/desktop/src/renderer/src/data/fixtures.ts @@ -361,11 +361,14 @@ export interface CalibrationData { scatter: number[][] fit: number effect: { raw: string; banded: string; p50: string } + /** Closed+estimated issues that can't calibrate (same-day / 0-day closes). */ + excludedSameDay: number } export const CALIBRATION: CalibrationData = { n: 27, active: true, + excludedSameDay: 0, labels: [ { label: 'est/1d', n: 8, median: '1.1d', bias: 8 }, { label: 'est/2d', n: 9, median: '2.4d', bias: 18 }, diff --git a/apps/desktop/src/renderer/src/lib/backlog.ts b/apps/desktop/src/renderer/src/lib/backlog.ts index e55bbea..795bc67 100644 --- a/apps/desktop/src/renderer/src/lib/backlog.ts +++ b/apps/desktop/src/renderer/src/lib/backlog.ts @@ -1,6 +1,8 @@ import { + type CalibrationCoverage, type CalibrationModel, type CalibrationSample, + calibrationCoverage, calibrationSamples, type CapacityMember, capacityPerWorkday, @@ -171,10 +173,11 @@ export function backlogCalibration( issues: GiteaIssue[], timelines: Timelines = {}, asOf: Date = new Date(), -): { model: CalibrationModel; data: CalibrationData } { +): { model: CalibrationModel; data: CalibrationData; coverage: CalibrationCoverage } { const samples = calibrationSamples(issues, timelines, asOf) const model = fitCalibration(samples) - return { model, data: calibrationData(model, samples, issues) } + const coverage = calibrationCoverage(issues, timelines, asOf) + return { model, coverage, data: calibrationData(model, samples, issues, coverage.excludedSameDay) } } const pctFromMu = (mu: number) => Math.round((Math.exp(mu) - 1) * 100) @@ -188,6 +191,7 @@ export function calibrationData( model: CalibrationModel, samples: CalibrationSample[], openIssues: GiteaIssue[], + excludedSameDay = 0, ): CalibrationData { const labels = PRIOR_BUCKETS.map((b) => { const inBucket = samples.filter((s) => s.bucket === b) @@ -227,6 +231,7 @@ export function calibrationData( scatter: samples.map((s) => [s.estimateDays, s.actualWorkingDays]), fit: Number(Math.exp(model.global.mu).toFixed(2)), effect, + excludedSameDay, } } diff --git a/packages/core/src/calibration/calibration-v0.test.ts b/packages/core/src/calibration/calibration-v0.test.ts index 09e8486..7dbfa0c 100644 --- a/packages/core/src/calibration/calibration-v0.test.ts +++ b/packages/core/src/calibration/calibration-v0.test.ts @@ -5,6 +5,7 @@ import type { LifecycleEvent } from '../lifecycle/lifecycle-v0.js' import type { GiteaIssue } from '../gitea/types.js' import { CALIBRATION_BUCKET_FLOOR, + calibrationCoverage, calibrationSamples, type CalibrationSample, COLD_START_THRESHOLD, @@ -118,4 +119,26 @@ describe('calibrationSamples', () => { const noEst = issue({ number: 9, labels: [] }) expect(calibrationSamples([open, noEst], { ...events(8), ...events(9) }, asOf)).toEqual([]) }) + + it('coverage counts same-day closes as excluded candidates, not as "more closes needed"', () => { + // usable: commit Wed 01-07 → close Mon 01-12 = 3 working days + const usable = issue({ number: 7, labels: ['est/2d'] }) + // same-day close: commit and close on the same day = 0 working days → excluded + const sameDay = issue({ number: 10, labels: ['est/2d'], createdAt: '2026-01-12T08:00:00Z' }) + const sameDayEvents = { + 10: [ + { type: 'commit', at: '2026-01-12T09:00:00Z' } as LifecycleEvent, + { type: 'close', at: '2026-01-12T17:00:00Z' } as LifecycleEvent, + ], + } + const open = issue({ number: 8, state: 'open', labels: ['est/2d'], closedAt: null }) + const noEst = issue({ number: 9, labels: [] }) + + const cov = calibrationCoverage([usable, sameDay, open, noEst], { ...events(7), ...sameDayEvents }, asOf) + expect(cov.candidates).toBe(2) // closed + estimated only (usable + sameDay) + expect(cov.usable).toBe(1) + expect(cov.excludedSameDay).toBe(1) + // the honest denominator: usable matches the model's n + expect(cov.usable).toBe(calibrationSamples([usable, sameDay, open, noEst], { ...events(7), ...sameDayEvents }, asOf).length) + }) }) diff --git a/packages/core/src/calibration/calibration-v0.ts b/packages/core/src/calibration/calibration-v0.ts index 75ec6fb..767e887 100644 --- a/packages/core/src/calibration/calibration-v0.ts +++ b/packages/core/src/calibration/calibration-v0.ts @@ -125,3 +125,40 @@ export function calibrationSamples( } return out } + +/** How the closed+estimated backlog splits into usable samples vs. what can't calibrate. */ +export interface CalibrationCoverage { + /** Closed issues carrying an estimate — the calibration candidates. */ + candidates: number + /** Candidates that yielded a usable actual (> 0 working days) → become samples. */ + usable: number + /** + * Candidates excluded because the issue closed with 0 working days (same-day + * close) or no resolvable actual — real closes that structurally can't + * calibrate. Counting them keeps `usable/threshold` honest: it's not "N more + * closes away" if some of your closes will never count. + */ + excludedSameDay: number +} + +/** + * Coverage of the calibration candidates — how many closed+estimated issues are + * usable vs. silently unusable (same-day / 0-day closes). {@link calibrationSamples} + * drops the latter; this counts them so the UI can say *why* the sample is thin. + */ +export function calibrationCoverage( + issues: GiteaIssue[], + timelines: Record, + asOf: Date, +): CalibrationCoverage { + let candidates = 0 + let usable = 0 + for (const issue of issues) { + if (issue.state !== 'closed') continue + if (issue.facts.estimateDays == null) continue + candidates++ + const inf = inferLifecycle(issue, timelines[issue.number] ?? [], asOf) + if (inf.actualWorkingDays != null && inf.actualWorkingDays > 0) usable++ + } + return { candidates, usable, excludedSameDay: candidates - usable } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f54a11d..840ac21 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -73,6 +73,7 @@ export type { export { CALIBRATION_BUCKET_FLOOR, + calibrationCoverage, calibrationSamples, COLD_START_THRESHOLD, fitCalibration, @@ -80,6 +81,7 @@ export { } from './calibration/calibration-v0.js' export type { BucketFit, + CalibrationCoverage, CalibrationModel, CalibrationSample, PersonBias, -- 2.49.1