calibration: count same-day closes honestly (#34)
The cold-start surface showed "N/20 closed issues estimated", implying you're
just (20−N) closes away. But calibrationSamples silently drops closed+estimated
issues that closed in 0 working days (same-day closes) — real closes that
structurally can't calibrate. On this repo that's 10 of 24 closes hidden: the
note read 14/20 as if 6 away, when a third of the history will never count.
- core: `calibrationCoverage(issues, timelines, asOf)` → { candidates, usable,
excludedSameDay }, counting the silently-excluded same-day closes. Pure, tested.
- surface it: CalibrationData gains `excludedSameDay`; backlogCalibration returns
the coverage; the Runway note and the Calibration screen now say "… · N same-day
closes can't calibrate" so the thin sample is explained, not just reported.
Verified on christian/commitea: closed=24, usable=14, excludedSameDay=10.
131 core green (incl. new coverage test); core + desktop typecheck; 14 fixture e2e.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import type { LifecycleEvent } from '../lifecycle/lifecycle-v0.js'
|
||||
import type { GiteaIssue } from '../gitea/types.js'
|
||||
import {
|
||||
CALIBRATION_BUCKET_FLOOR,
|
||||
calibrationCoverage,
|
||||
calibrationSamples,
|
||||
type CalibrationSample,
|
||||
COLD_START_THRESHOLD,
|
||||
@@ -118,4 +119,26 @@ describe('calibrationSamples', () => {
|
||||
const noEst = issue({ number: 9, labels: [] })
|
||||
expect(calibrationSamples([open, noEst], { ...events(8), ...events(9) }, asOf)).toEqual([])
|
||||
})
|
||||
|
||||
it('coverage counts same-day closes as excluded candidates, not as "more closes needed"', () => {
|
||||
// usable: commit Wed 01-07 → close Mon 01-12 = 3 working days
|
||||
const usable = issue({ number: 7, labels: ['est/2d'] })
|
||||
// same-day close: commit and close on the same day = 0 working days → excluded
|
||||
const sameDay = issue({ number: 10, labels: ['est/2d'], createdAt: '2026-01-12T08:00:00Z' })
|
||||
const sameDayEvents = {
|
||||
10: [
|
||||
{ type: 'commit', at: '2026-01-12T09:00:00Z' } as LifecycleEvent,
|
||||
{ type: 'close', at: '2026-01-12T17:00:00Z' } as LifecycleEvent,
|
||||
],
|
||||
}
|
||||
const open = issue({ number: 8, state: 'open', labels: ['est/2d'], closedAt: null })
|
||||
const noEst = issue({ number: 9, labels: [] })
|
||||
|
||||
const cov = calibrationCoverage([usable, sameDay, open, noEst], { ...events(7), ...sameDayEvents }, asOf)
|
||||
expect(cov.candidates).toBe(2) // closed + estimated only (usable + sameDay)
|
||||
expect(cov.usable).toBe(1)
|
||||
expect(cov.excludedSameDay).toBe(1)
|
||||
// the honest denominator: usable matches the model's n
|
||||
expect(cov.usable).toBe(calibrationSamples([usable, sameDay, open, noEst], { ...events(7), ...sameDayEvents }, asOf).length)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -125,3 +125,40 @@ export function calibrationSamples(
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** How the closed+estimated backlog splits into usable samples vs. what can't calibrate. */
|
||||
export interface CalibrationCoverage {
|
||||
/** Closed issues carrying an estimate — the calibration candidates. */
|
||||
candidates: number
|
||||
/** Candidates that yielded a usable actual (> 0 working days) → become samples. */
|
||||
usable: number
|
||||
/**
|
||||
* Candidates excluded because the issue closed with 0 working days (same-day
|
||||
* close) or no resolvable actual — real closes that structurally can't
|
||||
* calibrate. Counting them keeps `usable/threshold` honest: it's not "N more
|
||||
* closes away" if some of your closes will never count.
|
||||
*/
|
||||
excludedSameDay: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage of the calibration candidates — how many closed+estimated issues are
|
||||
* usable vs. silently unusable (same-day / 0-day closes). {@link calibrationSamples}
|
||||
* drops the latter; this counts them so the UI can say *why* the sample is thin.
|
||||
*/
|
||||
export function calibrationCoverage(
|
||||
issues: GiteaIssue[],
|
||||
timelines: Record<number, LifecycleEvent[]>,
|
||||
asOf: Date,
|
||||
): CalibrationCoverage {
|
||||
let candidates = 0
|
||||
let usable = 0
|
||||
for (const issue of issues) {
|
||||
if (issue.state !== 'closed') continue
|
||||
if (issue.facts.estimateDays == null) continue
|
||||
candidates++
|
||||
const inf = inferLifecycle(issue, timelines[issue.number] ?? [], asOf)
|
||||
if (inf.actualWorkingDays != null && inf.actualWorkingDays > 0) usable++
|
||||
}
|
||||
return { candidates, usable, excludedSameDay: candidates - usable }
|
||||
}
|
||||
|
||||
@@ -73,6 +73,7 @@ export type {
|
||||
|
||||
export {
|
||||
CALIBRATION_BUCKET_FLOOR,
|
||||
calibrationCoverage,
|
||||
calibrationSamples,
|
||||
COLD_START_THRESHOLD,
|
||||
fitCalibration,
|
||||
@@ -80,6 +81,7 @@ export {
|
||||
} from './calibration/calibration-v0.js'
|
||||
export type {
|
||||
BucketFit,
|
||||
CalibrationCoverage,
|
||||
CalibrationModel,
|
||||
CalibrationSample,
|
||||
PersonBias,
|
||||
|
||||
Reference in New Issue
Block a user