feat: stream Reginald's replies token-by-token

The 26b is slow (~30s/call); the chat now shows the answer forming instead of
freezing until it's done. The final prose streams over SSE; tool-calling turns
stay structured (no partial tokens), so streaming kicks in for the narration.

core (@commitea/core):
- chat-client.complete gains an optional onToken — when set, it requests
  stream:true and parses the OpenAI SSE stream, emitting content deltas and
  assembling streamed tool-call argument fragments into the final result.
- GiteaHttpResponse exposes the optional `body` stream (real fetch has it; stubs
  don't). agent-loop threads onToken to each completion.

app:
- model:chat forwards each delta to the renderer (event.sender.send); preload
  exposes model.onToken(cb) → unsubscribe. useChat accumulates the live stream
  into a growing bubble (with a cursor), replaced by the authoritative final
  content when the turn resolves. Unconfigured → scripted reply, unchanged.

Verified: 118 core tests green (2 streaming: SSE content deltas + tool-call
fragment assembly), desktop typecheck clean, 14 fixture e2e green. Live: a real
turn against gemma-4-26b assembles the correct answer via the streaming path
(live-reginald green) — the reply now renders token-by-token.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Croissant Le Doux
2026-07-09 00:22:12 -04:00
parent 403aef9d16
commit dbcdcda5e7
10 changed files with 152 additions and 15 deletions

View File

@@ -48,8 +48,12 @@ export interface CompletionResult {
toolCalls: ToolCall[]
}
/** Called with each streamed content delta (final-prose streaming). */
export type OnToken = (delta: string) => void
export interface ChatClient {
complete(messages: ChatMessage[], tools?: ToolDecl[]): Promise<CompletionResult>
/** When `onToken` is given, the response streams (SSE) and each content delta is emitted. */
complete(messages: ChatMessage[], tools?: ToolDecl[], onToken?: OnToken): Promise<CompletionResult>
}
/** Map our message shape to the OpenAI wire shape. */
@@ -85,7 +89,7 @@ export function createChatClient(config: ModelConfig, fetchImpl: FetchLike): Cha
const url = `${config.baseUrl.replace(/\/+$/, '')}/chat/completions`
return {
async complete(messages, tools) {
async complete(messages, tools, onToken) {
const body: Record<string, unknown> = {
model: config.model,
messages: messages.map(toWireMessage),
@@ -95,11 +99,13 @@ export function createChatClient(config: ModelConfig, fetchImpl: FetchLike): Cha
body.tools = tools.map(toWireTool)
body.tool_choice = 'auto'
}
const stream = !!onToken
if (stream) body.stream = true
const res = await fetchImpl(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
Accept: stream ? 'text/event-stream' : 'application/json',
...(config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}),
},
body: JSON.stringify(body),
@@ -108,6 +114,8 @@ export function createChatClient(config: ModelConfig, fetchImpl: FetchLike): Cha
const text = await res.text().catch(() => '')
throw new Error(`model completion failed (${res.status}): ${text.slice(0, 200)}`)
}
if (stream && res.body) return readStream(res.body, onToken!)
const json = (await res.json()) as { choices?: { message: RawChoiceMessage }[] }
const msg = json.choices?.[0]?.message
return {
@@ -121,3 +129,56 @@ export function createChatClient(config: ModelConfig, fetchImpl: FetchLike): Cha
},
}
}
/** Streamed tool-call delta: name arrives first, arguments accumulate across chunks. */
interface RawToolDelta {
index: number
id?: string
function?: { name?: string; arguments?: string }
}
/** Parse an OpenAI SSE stream: emit content deltas via onToken, accumulate the final result. */
async function readStream(body: ReadableStream<Uint8Array>, onToken: OnToken): Promise<CompletionResult> {
const reader = body.getReader()
const decoder = new TextDecoder()
let buffer = ''
let content = ''
const toolAcc: { id: string; name: string; arguments: string }[] = []
const handle = (data: string) => {
if (data === '[DONE]') return
let chunk: { choices?: { delta?: { content?: string; tool_calls?: RawToolDelta[] } }[] }
try {
chunk = JSON.parse(data)
} catch {
return
}
const delta = chunk.choices?.[0]?.delta
if (!delta) return
if (delta.content) {
content += delta.content
onToken(delta.content)
}
for (const tc of delta.tool_calls ?? []) {
const slot = (toolAcc[tc.index] ??= { id: '', name: '', arguments: '' })
if (tc.id) slot.id = tc.id
if (tc.function?.name) slot.name = tc.function.name
if (tc.function?.arguments) slot.arguments += tc.function.arguments
}
}
for (;;) {
const { done, value } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
const lines = buffer.split('\n')
buffer = lines.pop() ?? ''
for (const line of lines) {
const t = line.trim()
if (t.startsWith('data:')) handle(t.slice(5).trim())
}
}
if (buffer.trim().startsWith('data:')) handle(buffer.trim().slice(5).trim())
return { content, toolCalls: toolAcc.filter((t) => t.name).map((t) => ({ id: t.id, name: t.name, arguments: t.arguments })) }
}