diff --git a/client/src/Providers/SideChatHostContext.tsx b/client/src/Providers/SideChatHostContext.tsx index 5a32abbd567..9b33bbce0c9 100644 --- a/client/src/Providers/SideChatHostContext.tsx +++ b/client/src/Providers/SideChatHostContext.tsx @@ -2,13 +2,19 @@ import { useCallback, useMemo } from 'react'; import { useStore } from 'jotai'; import { useQueryClient } from '@tanstack/react-query'; import { createGenerationTransport } from '@librechat/chat'; -import { filesFamily, useFollowChat, SIDE_CHAT_PANE } from '@librechat/chat/react'; import { dataService, QueryKeys, isAgentsEndpoint, isEphemeralAgentId, } from 'librechat-data-provider'; +import { + filesFamily, + useFollowChat, + SIDE_CHAT_PANE, + useUsageHandler, + toResumableUsage, +} from '@librechat/chat/react'; import type { Agent, TMessage, TConversation, ChatFinalFrame } from 'librechat-data-provider'; import type { FollowedChat, SideChatHost, FollowChatOptions } from '@librechat/chat/react'; import { isTemporaryConversation, proveFetchedReplyByIdentity } from '~/utils'; @@ -82,6 +88,13 @@ function useChat( * the app's file, memory and preview caches. */ const onAttachment = useAttachmentHandler(queryClient); + /** + * The conversation's token usage, kept by main chat's own writers: the context reading the + * server sends as a reply starts, the output arriving meanwhile, and the totals at its end. + * It is what the composer's gauge reads, by conversation. + */ + const usageHandlers = useUsageHandler(); + const usage = useMemo(() => toResumableUsage(usageHandlers), [usageHandlers]); /** * What lets a reply read here be acknowledged, as main chat's is: the reply a turn just ended * on is stamped on the conversation wherever it is listed, paired with the list that holds @@ -90,6 +103,8 @@ function useChat( const onSettled = useCallback( (final?: ChatFinalFrame) => { if (final == null) { + /** No reply is on its way in what was just read, so no estimate of one is kept. */ + usageHandlers.resetLive({ conversation: { conversationId: id } }); const messages = queryClient.getQueryData([QueryKeys.messages, id]); if (messages != null) { proveFetchedReplyByIdentity(queryClient, id, messages); @@ -106,13 +121,13 @@ function useChat( updatedAt: settled.updatedAt, }); }, - [queryClient, id], + [queryClient, id, usageHandlers], ); /** A turn here is said to a screen reader as main chat's is: started, still going, done. */ const { announcePolite: announce } = useLiveAnnouncer(); const beside = useMemo( - () => ({ onAttachment, onSettled, announce }), - [onAttachment, onSettled, announce], + () => ({ onAttachment, onSettled, announce, usage }), + [onAttachment, onSettled, announce, usage], ); return useFollowChat({ id, diff --git a/client/src/Providers/__tests__/SideChatHostContext.spec.tsx b/client/src/Providers/__tests__/SideChatHostContext.spec.tsx index 8f715e647cf..81ca73245be 100644 --- a/client/src/Providers/__tests__/SideChatHostContext.spec.tsx +++ b/client/src/Providers/__tests__/SideChatHostContext.spec.tsx @@ -1,5 +1,7 @@ import React from 'react'; import { RecoilRoot } from 'recoil'; +import { getDefaultStore } from 'jotai'; +import { contextSnapshotFamily } from '@librechat/chat/react'; import { act, renderHook, waitFor } from '@testing-library/react'; import { QueryKeys, EModelEndpoint } from 'librechat-data-provider'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; @@ -364,6 +366,49 @@ describe("the app's side chat host", () => { expect(announcePolite).toHaveBeenCalledWith({ message: 'end', isStatus: true }); }); + it("keeps the conversation's usage as main chat keeps it, for the composer's gauge", async () => { + mockStreamStatus.mockResolvedValue({ + active: true, + streamId: 'side-conversation', + createdAt: 2000, + generationProtocolVersion: 2, + }); + mockMessages.mockResolvedValue([asked]); + const { attachments } = setup(); + await waitFor(() => expect(attachments).toHaveLength(1)); + const emit = (event: ChatEvent) => act(() => attachments[0].options.onEvent(event)); + const usage = getDefaultStore(); + expect(usage.get(contextSnapshotFamily('side-conversation'))).toBeNull(); + + emit({ type: 'open' }); + emit({ + type: 'created', + data: { created: true, message: asked, turn: { responseMessageId: 'response-1' } }, + } as ChatEvent); + /** What the server reads the context at as the reply starts. */ + emit({ + type: 'context_usage', + data: { + breakdown: { + maxContextTokens: 8000, + instructionTokens: 120, + systemMessageTokens: 100, + dynamicInstructionTokens: 0, + toolSchemaTokens: 20, + summaryTokens: 0, + toolCount: 1, + messageCount: 1, + messageTokens: 340, + availableForMessages: 7880, + }, + }, + } as ChatEvent); + + await waitFor(() => + expect(usage.get(contextSnapshotFamily('side-conversation'))).not.toBeNull(), + ); + }); + it('stops a generation through the app, by its conversation and the stream it names', async () => { mockStreamStatus.mockResolvedValue({ active: true, diff --git a/client/src/hooks/SSE/events.ts b/client/src/hooks/SSE/events.ts index f30adb923fa..a587388bd62 100644 --- a/client/src/hooks/SSE/events.ts +++ b/client/src/hooks/SSE/events.ts @@ -2,12 +2,17 @@ import { useMemo, useState, useCallback } from 'react'; import { useSetAtom } from 'jotai'; import { useQueryClient } from '@tanstack/react-query'; import { getStreamStartFailureText } from '@librechat/chat'; -import { useUsageHandler, useEventHandlers, showStopButtonByIndex } from '@librechat/chat/react'; +import { + useUsageHandler, + toResumableUsage, + useEventHandlers, + showStopButtonByIndex, +} from '@librechat/chat/react'; import type { ResumableRun, ResumableEvents, ResumableUsage } from '@librechat/chat'; import type { EventHandlerParams } from '@librechat/chat/react'; import type { EventSubmission } from 'librechat-data-provider'; import type { TResData, TFinalResData } from '~/common'; -import { countTrailingOutputChars, markStreamStartFailedMetadata } from '~/utils'; +import { markStreamStartFailedMetadata } from '~/utils'; import { reopenTermsOnRefusal } from '~/utils/terms'; import useLocalize from '~/hooks/useLocalize'; @@ -93,17 +98,8 @@ export default function useResumableEvents({ [rawStepHandler, run], ); - const { - bindResponse, - contextHandler, - usageHandler, - tapStream, - tapContent, - finalizeUsage, - backfillUsage, - resetLive, - seedLive, - } = useUsageHandler(); + const usageHandlers = useUsageHandler(); + const { resetLive } = usageHandlers; const events = useMemo( () => ({ @@ -158,29 +154,7 @@ export default function useResumableEvents({ ], ); - const usage = useMemo( - () => ({ - bindResponse, - contextHandler, - usageHandler, - tapStream, - tapContent, - finalizeUsage, - backfillUsage, - seedLiveFromContent: (content, usageSubmission) => - seedLive(countTrailingOutputChars(content), usageSubmission), - }), - [ - bindResponse, - contextHandler, - usageHandler, - tapStream, - tapContent, - finalizeUsage, - backfillUsage, - seedLive, - ], - ); + const usage = useMemo(() => toResumableUsage(usageHandlers), [usageHandlers]); return { events, usage }; } diff --git a/e2e/specs/mock/scenarios/gauge-follows-reply.spec.ts b/e2e/specs/mock/scenarios/gauge-follows-reply.spec.ts new file mode 100644 index 00000000000..70316cdf82d --- /dev/null +++ b/e2e/specs/mock/scenarios/gauge-follows-reply.spec.ts @@ -0,0 +1,66 @@ +import { expect, test } from '@playwright/test'; +import { + replyText, + replyPrompt, + messagesView, + NEW_CHAT_PATH, + MOCK_ENDPOINTS, + selectMockEndpoint, + sendMessageAndWaitForCompletion, +} from '../helpers'; + +/** + * The context gauge counts a reply as it is written: the reading the server sends when the + * reply starts, plus the output that has arrived since. + */ +test.use({ viewport: { width: 1440, height: 900 }, isMobile: false, hasTouch: false }); + +const uniqueLabel = (prefix: string) => + `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; + +/** The gauge's used-token count, read from its accessible name. */ +const used = (label: string | null): number => { + const match = /Context window: ([\d.,]+)(K?) of/.exec(label ?? ''); + if (match == null) { + throw new Error(`The gauge does not say how much is used: ${label}`); + } + return Number(match[1].replace(/,/g, '')) * (match[2] === 'K' ? 1000 : 1); +}; + +test('the gauge keeps counting while a reply streams @scenario:context-gauge-follows-a-streaming-reply', async ({ + page, +}) => { + test.setTimeout(120_000); + const label = uniqueLabel('gauge'); + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + const first = await sendMessageAndWaitForCompletion(page, replyPrompt(label)); + expect(first.ok()).toBeTruthy(); + await expect(messagesView(page).getByText(replyText(label))).toBeVisible(); + + const form = page.locator('main form').first(); + const gauge = form.getByTestId('token-usage'); + await form.getByRole('textbox', { name: 'Message input' }).click(); + await page.keyboard.type(`E2E_SLOW_REPLY:${label}`); + await page.keyboard.press('Enter'); + + /** + * Early in the reply, and again well into it, both while it is still being written: more has + * arrived, so more is used. Read as they stand, with no waiting that could run past the end + * of the reply, where the count settles whatever it did meanwhile. + */ + const stop = page.getByRole('button', { name: 'Stop generating' }); + await expect(messagesView(page)).toContainText('chunk-020', { timeout: 30_000 }); + const early = used(await gauge.getAttribute('aria-label')); + await expect(messagesView(page)).toContainText('chunk-100', { timeout: 60_000 }); + const late = used(await gauge.getAttribute('aria-label')); + await expect(stop).toBeVisible(); + expect(late).toBeGreaterThan(early); + + /** The settled count is not below what it showed on the way. */ + await expect(messagesView(page)).toContainText('chunk-159', { timeout: 60_000 }); + await expect(stop).toHaveCount(0); + await expect + .poll(async () => used(await gauge.getAttribute('aria-label')), { timeout: 10_000 }) + .toBeGreaterThanOrEqual(late - 5); +}); diff --git a/e2e/specs/mock/scenarios/side-chat-composer.spec.ts b/e2e/specs/mock/scenarios/side-chat-composer.spec.ts index a3299a7e85a..4db82be3b0f 100644 --- a/e2e/specs/mock/scenarios/side-chat-composer.spec.ts +++ b/e2e/specs/mock/scenarios/side-chat-composer.spec.ts @@ -375,3 +375,37 @@ for (const saveDrafts of [true, false]) { await expect(main.getByText('comes-along.txt')).toHaveCount(1); }); } + +/** The gauge's used-token count, read from its accessible name. */ +const usedTokens = (label: string | null): number => { + const match = /Context window: ([\d.,]+)(K?) of/.exec(label ?? ''); + if (match == null) { + throw new Error(`The gauge does not say how much is used: ${label}`); + } + return Number(match[1].replace(/,/g, '')) * (match[2] === 'K' ? 1000 : 1); +}; + +test('the gauge beside the chat counts a reply as it is written @scenario:side-chat-composer-gauge-follows-the-reply', async ({ + page, +}) => { + test.setTimeout(120_000); + const { sideChat, side, field } = await openBeside(page, 'gauge'); + const gauge = side.getByTestId('token-usage'); + + await field.click(); + await page.keyboard.type(`E2E_SLOW_REPLY:${uniqueLabel('measured')}`); + await page.keyboard.press('Enter'); + + /** + * Two readings while the reply is still running, as main chat's gauge gives them: the count + * the server sends as the reply starts, plus what has arrived since. No waiting between that + * could run past the end of the reply, where the count settles whatever it did meanwhile. + */ + const stop = side.getByRole('button', { name: 'Stop generating' }); + await expect(sideChat).toContainText('chunk-020', { timeout: 30_000 }); + const early = usedTokens(await gauge.getAttribute('aria-label')); + await expect(sideChat).toContainText('chunk-100', { timeout: 60_000 }); + const late = usedTokens(await gauge.getAttribute('aria-label')); + await expect(stop).toBeVisible(); + expect(late).toBeGreaterThan(early); +}); diff --git a/packages/chat/src/core/__tests__/nameStepResponse.spec.ts b/packages/chat/src/core/__tests__/nameStepResponse.spec.ts new file mode 100644 index 00000000000..12c47cc01c0 --- /dev/null +++ b/packages/chat/src/core/__tests__/nameStepResponse.spec.ts @@ -0,0 +1,41 @@ +import { Constants, StepEvents } from 'librechat-data-provider'; +import type { TSubmission } from 'librechat-data-provider'; +import type { StepEvent } from '../reducer'; +import { nameStepResponse } from '../resumable/events'; + +const submission = { userMessage: { messageId: 'u1' } } as TSubmission; +const runStep = (runId: string | undefined): StepEvent => + ({ event: StepEvents.ON_RUN_STEP, data: { id: 's0', runId, index: 0 } }) as StepEvent; + +describe('nameStepResponse', () => { + it('tells the usage writers the reply id a run step carries', () => { + const nameResponse = jest.fn(); + + nameStepResponse({ nameResponse }, runStep('stored-reply'), submission); + + expect(nameResponse).toHaveBeenCalledWith('stored-reply', submission); + }); + + it('says nothing for a step that names no row of its own, or for any other event', () => { + const nameResponse = jest.fn(); + + nameStepResponse({ nameResponse }, runStep(undefined), submission); + nameStepResponse( + { nameResponse }, + runStep(Constants.USE_PRELIM_RESPONSE_MESSAGE_ID as string), + submission, + ); + nameStepResponse( + { nameResponse }, + { event: StepEvents.ON_MESSAGE_DELTA, data: { id: 's0', delta: {} } } as StepEvent, + submission, + ); + + expect(nameResponse).not.toHaveBeenCalled(); + }); + + it('is harmless where no usage is kept', () => { + expect(() => nameStepResponse(undefined, runStep('stored-reply'), submission)).not.toThrow(); + expect(() => nameStepResponse({}, runStep('stored-reply'), submission)).not.toThrow(); + }); +}); diff --git a/packages/chat/src/core/apply.ts b/packages/chat/src/core/apply.ts index 17b1ee51bb9..da7d0d3dc99 100644 --- a/packages/chat/src/core/apply.ts +++ b/packages/chat/src/core/apply.ts @@ -31,7 +31,7 @@ import { placeReasoningLabel, } from './parts/place'; import { buildResumeEventSubmission, buildSubmissionFromResumeState } from './resumable/resume'; -import { getStepEventId, isOAuthStepEvent } from './resumable/events'; +import { getStepEventId, isOAuthStepEvent, nameStepResponse } from './resumable/events'; import { buildCreatedInitialResponse } from './resumable/messages'; import { placeResumedResponse } from './resumable/placement'; import { normalizeFrame } from './transport/frames'; @@ -520,6 +520,7 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { if (isOutputDelta(event)) { init.usage?.tapStream(event.data as { delta?: { content?: unknown } }, submission); } + nameStepResponse(init.usage, event, submission); settle(reduceStep(stream, event, stepContext(submission, context))); }; diff --git a/packages/chat/src/core/resumable/events.ts b/packages/chat/src/core/resumable/events.ts index e02c765dbe7..b6e8110b538 100644 --- a/packages/chat/src/core/resumable/events.ts +++ b/packages/chat/src/core/resumable/events.ts @@ -1,4 +1,7 @@ import { Constants, StepEvents } from 'librechat-data-provider'; +import type { TSubmission } from 'librechat-data-provider'; +import type { ResumableUsage } from './host'; +import type { StepEvent } from '../reducer'; const getToolCallName = (toolCall: unknown): unknown => toolCall != null && typeof toolCall === 'object' && 'name' in toolCall @@ -74,3 +77,24 @@ export const getStepEventId = (event: unknown): string | undefined => { ? result.id : undefined; }; + +/** + * Tells the usage writers the id a run step gives the turn's reply. The reply is drawn under a + * placeholder id until then, and usage is bound to the submission that still names the + * placeholder; without this the reply being written stops counting as the one in progress the + * moment its row is renamed, and what a reader sees of its usage stands still until it ends. + */ +export function nameStepResponse( + usage: Pick | undefined, + event: StepEvent, + submission: TSubmission, +): void { + if (event.event !== StepEvents.ON_RUN_STEP) { + return; + } + const { runId } = event.data; + if (!runId || runId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { + return; + } + usage?.nameResponse?.(runId, submission); +} diff --git a/packages/chat/src/core/resumable/frames.ts b/packages/chat/src/core/resumable/frames.ts index b83928a2223..40038e6d1d0 100644 --- a/packages/chat/src/core/resumable/frames.ts +++ b/packages/chat/src/core/resumable/frames.ts @@ -17,9 +17,9 @@ import { } from './terminal'; import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol'; import { isInitialNewConversation, buildCreatedInitialResponse } from './messages'; +import { isOAuthStepEvent, nameStepResponse } from './events'; import { reconcileGenerationLifecycle } from './reconcile'; import { ABORT_SWEEP_STATUSES } from '../steer/chips'; -import { isOAuthStepEvent } from './events'; import { handleSync } from './sync'; export function handleOpen({ sub, host }: StreamContext): void { @@ -298,6 +298,7 @@ export async function handleFrame(ctx: StreamContext, event: ChatEvent): Promise sub.preCreatedStepEvents.push(data); return; } + nameStepResponse(host.usage, data, sub.liveSubmission()); host.events.stepHandler(data, sub.liveSubmission() as EventSubmission); return; } diff --git a/packages/chat/src/core/resumable/host.ts b/packages/chat/src/core/resumable/host.ts index 9bc8156e54e..25cdd9ee1bf 100644 --- a/packages/chat/src/core/resumable/host.ts +++ b/packages/chat/src/core/resumable/host.ts @@ -303,6 +303,8 @@ export interface ResumableEvents { /** The run's token usage: the live estimate while it streams and the totals it settles to. */ export interface ResumableUsage { bindResponse: (submission: TSubmission) => void; + /** A run step named the id the turn's reply row takes; see `nameStepResponse`. */ + nameResponse?: (responseId: string, submission: TSubmission) => void; contextHandler: (data: TContextUsageEvent, submission: TSubmission) => void; usageHandler: (data: TTokenUsageEvent, submission: TSubmission) => void; tapStream: (data: { delta?: { content?: unknown } }, submission: TSubmission) => void; @@ -311,6 +313,11 @@ export interface ResumableUsage { backfillUsage: (entries: TTokenUsageEvent[], submission: TSubmission) => void; /** Seeds the live estimate with the output a resume snapshot already holds. */ seedLiveFromContent: (content: unknown[] | undefined, submission: TSubmission) => void; + /** + * Forgets what was counted for a reply that ended with no final frame to settle it. Called by + * what follows a core `Chat`; main chat's session does the same through its events. + */ + resetLive?: (submission: TSubmission) => void; } /** The pane's message list. */ diff --git a/packages/chat/src/core/resumable/patches.ts b/packages/chat/src/core/resumable/patches.ts index 1afb20f206c..75409b55616 100644 --- a/packages/chat/src/core/resumable/patches.ts +++ b/packages/chat/src/core/resumable/patches.ts @@ -16,6 +16,7 @@ import { placePendingAction, placeReasoningLabel, } from '../parts/place'; +import { nameStepResponse } from './events'; /** * Keep retrying across frames (not just once): message updates from the preceding created/step @@ -45,6 +46,7 @@ export class ResponsePatcher { const submission = sub.liveSubmission() as EventSubmission; for (const event of sub.preCreatedStepEvents.splice(0)) { + nameStepResponse(this.host.usage, event, submission); this.host.events.stepHandler(event, submission); } } diff --git a/packages/chat/src/react/chat/__tests__/driven.spec.tsx b/packages/chat/src/react/chat/__tests__/driven.spec.tsx index 6bc773e938a..c887634da81 100644 --- a/packages/chat/src/react/chat/__tests__/driven.spec.tsx +++ b/packages/chat/src/react/chat/__tests__/driven.spec.tsx @@ -17,8 +17,11 @@ import type { ChatStream, ChatTransport, ChatSendRequest, ChatStreamStatus } fro import type { ChatHost, TurnServices } from '../host'; import type { FollowChatOptions } from '../../follow'; import type { ExtendedFile } from '../../../core'; +import { createUsageHandlers, createUsageCounters } from '../../stream/usage/handlers'; +import { activeUsageResponseIdFamily, pendingUsageFamily } from '../../usage'; import { createTurnServices } from '../__fixtures__/services'; import { showStopButtonByIndex } from '../../generation'; +import { toResumableUsage } from '../../stream/usage'; import { isSubmittingFamily } from '../../submission'; import { useDrivenContract } from '../driven'; import { useFollowChat } from '../../follow'; @@ -382,6 +385,59 @@ describe('useDrivenContract', () => { expect(onSettled).toHaveBeenCalledTimes(2); }); + it('drops what was counted for a reply that fails, and keeps it for one that ends', async () => { + const store = createStore(); + const usage = toResumableUsage(createUsageHandlers(store, createUsageCounters())); + const { result, streams, ready } = setup({ store, beside: { usage } }); + await ready(); + const spent = (output_tokens: number, seq: number): ChatEvent => + ({ + type: 'token_usage', + data: { input_tokens: 10, output_tokens, runId: 'run-2', seq }, + }) as ChatEvent; + + act(() => { + result.current.contract.ask({ text: 'Next question' }); + }); + await waitFor(() => expect(streams).toHaveLength(1)); + const sent = (result.current.contract.getMessages() ?? [])[2]; + act(() => { + streams[0].push(created(sent, 'response-2')); + streams[0].push(spent(40, 0)); + }); + await waitFor(() => expect(store.get(pendingUsageFamily(conversationId)).output).toBe(40)); + expect(store.get(activeUsageResponseIdFamily(conversationId))).not.toBeNull(); + + act(() => streams[0].push({ type: 'error', status: 500, data: { message: 'failed' } })); + await waitFor(() => expect(result.current.contract.isSubmitting).toBe(false)); + await settle(); + + /** Nothing of the failed reply is left to be added to the conversation's next one. */ + expect(store.get(pendingUsageFamily(conversationId))).toMatchObject({ + output: 0, + eventCount: 0, + }); + expect(store.get(activeUsageResponseIdFamily(conversationId))).toBeNull(); + + /** A reply that ends on its final frame is settled by that frame instead. */ + const resetLive = jest.spyOn(usage, 'resetLive'); + act(() => { + result.current.contract.ask({ text: 'Again' }); + }); + await waitFor(() => expect(streams).toHaveLength(2)); + const again = (result.current.contract.getMessages() ?? []).find( + (message) => message.text === 'Again', + ) as TMessage; + act(() => { + streams[1].push(created(again, 'response-3')); + streams[1].push(spent(25, 1)); + streams[1].push(final(again, reply('response-3', 'Answer', again.messageId))); + }); + await waitFor(() => expect(result.current.contract.isSubmitting).toBe(false)); + await settle(); + expect(resetLive).not.toHaveBeenCalled(); + }); + it("drops the files another conversation left staged in the pane, and keeps this one's", async () => { const staged = new Map([['file-1', { file_id: 'file-1', progress: 1 } as ExtendedFile]]); const first = setup(); diff --git a/packages/chat/src/react/follow.ts b/packages/chat/src/react/follow.ts index 90612c832b5..255b31d46cf 100644 --- a/packages/chat/src/react/follow.ts +++ b/packages/chat/src/react/follow.ts @@ -102,6 +102,7 @@ export type FollowBeside = AppliedBeside & { const forwardBeside = (read: () => FollowBeside | undefined): Required => ({ usage: { bindResponse: (submission) => read()?.usage?.bindResponse(submission), + nameResponse: (responseId, submission) => read()?.usage?.nameResponse?.(responseId, submission), contextHandler: (data, submission) => read()?.usage?.contextHandler(data, submission), usageHandler: (data, submission) => read()?.usage?.usageHandler(data, submission), tapStream: (data, submission) => read()?.usage?.tapStream(data, submission), @@ -110,6 +111,7 @@ const forwardBeside = (read: () => FollowBeside | undefined): Required read()?.usage?.backfillUsage(entries, submission), seedLiveFromContent: (content, submission) => read()?.usage?.seedLiveFromContent(content, submission), + resetLive: (submission) => read()?.usage?.resetLive?.(submission), }, onAttachment: (params) => read()?.onAttachment?.(params), onTitle: (frame) => read()?.onTitle?.(frame), @@ -671,7 +673,14 @@ export function useFollowChat({ /** The turn's own final frame is its result, and the list as it stands is what it left. */ if (ended != null) { settled(); + return; } + /** + * It ended with no final frame: it failed, or was stopped before the server kept anything + * of it. Nothing settles what was counted for its reply, so that count is dropped here, as + * main chat drops it, instead of being added to the conversation's next reply. + */ + host.current.beside?.usage?.resetLive?.(turn.submission); }; /** diff --git a/packages/chat/src/react/stream/index.ts b/packages/chat/src/react/stream/index.ts index 854d2a4f602..d3116c43386 100644 --- a/packages/chat/src/react/stream/index.ts +++ b/packages/chat/src/react/stream/index.ts @@ -2,7 +2,7 @@ export { default as useAdaptiveSSE } from './adaptive'; export { default as useResumableSSE } from './resumable'; export { StreamHostProvider, useStreamHost } from './host'; export { useStepHandler } from './step'; -export { useUsageHandler } from './usage'; +export { useUsageHandler, toResumableUsage } from './usage'; export { useContentHandler } from './content'; export { settleFinishedRun } from './resume/finished'; export { useResumeAnnouncements } from './resume/announcements'; diff --git a/packages/chat/src/react/stream/usage/__tests__/handlers.spec.ts b/packages/chat/src/react/stream/usage/__tests__/handlers.spec.ts index a6f50b31a54..c44ef9e9e7a 100644 --- a/packages/chat/src/react/stream/usage/__tests__/handlers.spec.ts +++ b/packages/chat/src/react/stream/usage/__tests__/handlers.spec.ts @@ -23,6 +23,41 @@ describe('createUsageHandlers', () => { expect(store.get(pendingUsageFamily('c1')).eventCount).toBe(1); }); + /** The reply is drawn under a placeholder id and the first run step renames its row. Every + * later frame is handled with the submission, which still names the placeholder, so without + * the name the response in progress snapped back to an id no row has. */ + it('follows the reply to the id the stream names for it, through the frames after', () => { + const store = createStore(); + const handlers = createUsageHandlers(store, createUsageCounters()); + handlers.bindResponse(submission); + expect(store.get(activeUsageResponseIdFamily('c1'))).toBe('r1'); + + handlers.nameResponse('stored-reply', submission); + expect(store.get(activeUsageResponseIdFamily('c1'))).toBe('stored-reply'); + + handlers.tapStream({ delta: { content: [{ type: 'text', text: 'more words' }] } }, submission); + handlers.usageHandler(usage({}), submission); + expect(store.get(activeUsageResponseIdFamily('c1'))).toBe('stored-reply'); + expect(handlers.bindResponse(submission)).toBe('stored-reply'); + }); + + it('names nothing for another turn, whose reply has a placeholder of its own', () => { + const store = createStore(); + const handlers = createUsageHandlers(store, createUsageCounters()); + handlers.nameResponse('stored-reply', submission); + const next: UsageSubmissionLike = { + ...submission, + userMessage: { messageId: 'u2', conversationId: 'c1' }, + initialResponse: { messageId: 'r2', parentMessageId: 'u2' }, + }; + + expect(handlers.bindResponse(next)).toBe('r2'); + /** A name equal to the placeholder, or none at all, is no rename. */ + handlers.nameResponse('r2', next); + handlers.nameResponse('', next); + expect(store.get(activeUsageResponseIdFamily('c1'))).toBe('r2'); + }); + it('resetLive clears pending usage and the live estimate', () => { const store = createStore(); const handlers = createUsageHandlers(store, createUsageCounters()); diff --git a/packages/chat/src/react/stream/usage/events.ts b/packages/chat/src/react/stream/usage/events.ts index df0004478fd..1417e7a3124 100644 --- a/packages/chat/src/react/stream/usage/events.ts +++ b/packages/chat/src/react/stream/usage/events.ts @@ -23,6 +23,12 @@ export interface FinalDataLike { export interface UsageHandlers { /** Bind an optimistic, created, or resumed response without inventing usage. */ bindResponse: (submission: UsageSubmissionLike) => string | null; + /** + * The turn's reply was drawn under a placeholder id and the stream has now named the id its + * row takes. Usage bound to that submission follows the row from here on, so the reply being + * written stays the one in progress. + */ + nameResponse: (responseId: string, submission: UsageSubmissionLike) => void; contextHandler: (data: TContextUsageEvent, submission: UsageSubmissionLike) => void; usageHandler: (data: TTokenUsageEvent, submission: UsageSubmissionLike) => void; tapStream: (data: { delta?: { content?: unknown } }, submission: UsageSubmissionLike) => void; diff --git a/packages/chat/src/react/stream/usage/handlers.ts b/packages/chat/src/react/stream/usage/handlers.ts index 73ed7c71e98..f2a3ff5a073 100644 --- a/packages/chat/src/react/stream/usage/handlers.ts +++ b/packages/chat/src/react/stream/usage/handlers.ts @@ -93,13 +93,24 @@ export function createUsageHandlers( /** The transport owns response-id normalization. Use the same response it * renders, including durable IDs assigned by legacy text and resume events. */ + /** + * A turn draws its reply under a placeholder id, and the first run step renames the row to + * the id the server stores it under. The submission keeps the placeholder (other handlers + * key on it), so the id the row took is remembered here, by placeholder. + */ + const named = new Map(); + const renderedId = (submission: UsageSubmissionLike): string | null => { + const drawn = submission.initialResponse?.messageId ?? null; + return drawn == null ? null : (named.get(drawn) ?? drawn); + }; + const bindResponse: UsageHandlers['bindResponse'] = (submission) => { const convoKey = getConvoKey(submission); if (liveConversationKey === Constants.NEW_CONVO && convoKey !== liveConversationKey) { moveConversation(liveConversationKey, convoKey); } liveConversationKey = convoKey; - const responseId = submission.initialResponse?.messageId ?? null; + const responseId = renderedId(submission); const ownerAtom = activeUsageResponseIdFamily(convoKey); const previousId = jotai.get(ownerAtom); jotai.set(ownerAtom, responseId); @@ -118,6 +129,17 @@ export function createUsageHandlers( return responseId; }; + const nameResponse: UsageHandlers['nameResponse'] = (responseId, submission) => { + const drawn = submission.initialResponse?.messageId; + if (!drawn || !responseId || drawn === responseId) { + return; + } + /** One turn in flight per handler set, so earlier turns' names are no longer needed. */ + named.clear(); + named.set(drawn, responseId); + bindResponse(submission); + }; + const setLive = (convoKey: string, value: number) => { jotai.set(liveTokensFamily(convoKey), value); }; @@ -475,6 +497,7 @@ export function createUsageHandlers( return { bindResponse, + nameResponse, contextHandler, usageHandler, tapStream, diff --git a/packages/chat/src/react/stream/usage/index.ts b/packages/chat/src/react/stream/usage/index.ts index cfc900031b8..1d067f74891 100644 --- a/packages/chat/src/react/stream/usage/index.ts +++ b/packages/chat/src/react/stream/usage/index.ts @@ -1,7 +1,9 @@ import { useRef, useMemo } from 'react'; import { getDefaultStore } from 'jotai'; +import type { ResumableUsage } from '../../../core/resumable/host'; import type { UsageHandlers } from './handlers'; import { createUsageCounters, createUsageHandlers } from './handlers'; +import { countTrailingOutputChars } from '../../../core'; export type { UsageHandlers }; @@ -13,3 +15,24 @@ export function useUsageHandler(): UsageHandlers { const countersRef = useRef(createUsageCounters()); return useMemo(() => createUsageHandlers(getDefaultStore(), countersRef.current), []); } + +/** + * The usage writers as whatever applies a stream's frames calls them: main chat's stream + * session, and a core `Chat`'s event applier. One mapping, so both keep the same usage state + * for the same frames. + */ +export function toResumableUsage(handlers: UsageHandlers): ResumableUsage { + return { + bindResponse: handlers.bindResponse, + nameResponse: handlers.nameResponse, + contextHandler: handlers.contextHandler, + usageHandler: handlers.usageHandler, + tapStream: handlers.tapStream, + tapContent: handlers.tapContent, + finalizeUsage: handlers.finalizeUsage, + backfillUsage: handlers.backfillUsage, + seedLiveFromContent: (content, submission) => + handlers.seedLive(countTrailingOutputChars(content), submission), + resetLive: handlers.resetLive, + }; +}