diff --git a/packages/chat/src/core/__tests__/apply.spec.ts b/packages/chat/src/core/__tests__/apply.spec.ts new file mode 100644 index 00000000000..8bf918fd746 --- /dev/null +++ b/packages/chat/src/core/__tests__/apply.spec.ts @@ -0,0 +1,569 @@ +import { Constants, StepTypes, StepEvents, ContentTypes } from 'librechat-data-provider'; +import type { + Agents, + TMessage, + ChatEvent, + ChatStreamRequest, + ChatTransportOptions, + ChatTransport as WireTransport, +} from 'librechat-data-provider'; +import type { ChatEventContext, ChatState } from '../chat'; +import type { ChatStatus } from '../types'; +import { createGenerationTransport } from '../transport'; +import { createEventApplier } from '../apply'; +import { Chat } from '../chat'; + +const user: TMessage = { + messageId: 'user-1', + conversationId: 'thread-1', + parentMessageId: Constants.NO_PARENT as string, + isCreatedByUser: true, + text: 'Go', + sender: 'User', +}; + +const createList = (initial: TMessage[] = []) => { + let messages = initial; + const writes: TMessage[][] = []; + const context: ChatEventContext = { + get messages() { + return messages; + }, + write(next) { + messages = next; + writes.push(next); + }, + }; + return { context, writes, read: () => messages }; +}; + +const runStep = (id: string, index: number): Agents.RunStep => ({ + id, + runId: 'resp-1', + index, + type: StepTypes.MESSAGE_CREATION, + stepDetails: { type: StepTypes.MESSAGE_CREATION, message_creation: { message_id: id } }, + usage: null, +}); + +const opened = (id: string, index: number): ChatEvent => + ({ + type: 'step', + data: { event: StepEvents.ON_RUN_STEP, data: runStep(id, index) }, + }) as ChatEvent; + +const delta = (id: string, text: string) => ({ + event: StepEvents.ON_MESSAGE_DELTA, + data: { id, delta: { content: [{ type: ContentTypes.TEXT, text }] } }, +}); + +const token = (id: string, text: string): ChatEvent => + ({ type: 'step', data: delta(id, text) }) as ChatEvent; + +const created: ChatEvent = { type: 'created', data: { created: true, message: user } }; + +/** What the server sends a chat that attaches after `text` was already streamed. */ +const snapshot = (text: string, extra: Partial = {}): ChatEvent => + ({ + type: 'sync', + data: { + sync: true, + resumeState: { + conversationId: 'thread-1', + responseMessageId: 'resp-1', + sender: 'Researcher', + userMessage: { + messageId: 'user-1', + parentMessageId: Constants.NO_PARENT, + conversationId: 'thread-1', + text: 'Go', + }, + runSteps: [runStep('s0', 0)], + aggregatedContent: text === '' ? [] : [{ type: ContentTypes.TEXT, text }], + ...extra, + }, + }, + }) as ChatEvent; + +const final = (text: string): ChatEvent => + ({ + type: 'final', + data: { + final: true, + conversation: { conversationId: 'thread-1' }, + requestMessage: user, + responseMessage: { + messageId: 'resp-1', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + content: [{ type: ContentTypes.TEXT, text }], + }, + }, + }) as ChatEvent; + +const response = (messages: TMessage[]) => + messages.find((message) => message.messageId === 'resp-1'); + +const textOf = (messages: TMessage[]) => + (response(messages)?.content ?? []).map((part) => (part as { text?: string } | undefined)?.text); + +const shape = (messages: TMessage[]) => + messages.map(({ messageId, parentMessageId, isCreatedByUser, content }) => ({ + messageId, + parentMessageId, + isCreatedByUser, + content, + })); + +describe('createEventApplier', () => { + it('rebuilds the turn from a snapshot when the list holds none of it', () => { + const { applyEvent } = createEventApplier(); + const { context, read } = createList(); + + applyEvent(snapshot('Hel'), context); + + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-1']); + expect(read()[0]).toMatchObject({ isCreatedByUser: true, text: 'Go' }); + expect(response(read())).toMatchObject({ + parentMessageId: 'user-1', + isCreatedByUser: false, + sender: 'Researcher', + }); + expect(textOf(read())).toEqual(['Hel']); + }); + + it('continues the snapshot with the tokens that follow it', () => { + const { applyEvent } = createEventApplier(); + const { context, read } = createList(); + + applyEvent(snapshot('Hel'), context); + applyEvent(token('s0', 'lo'), context); + applyEvent(opened('s1', 1), context); + applyEvent(token('s1', ' there'), context); + + expect(textOf(read())).toEqual(['Hello', ' there']); + }); + + it('leaves a chat that joined late holding what one that watched from the start holds', () => { + const watcher = createList([user]); + const joiner = createList([user]); + const watching = createEventApplier(); + const joining = createEventApplier(); + + for (const event of [created, opened('s0', 0), token('s0', 'Hel'), token('s0', 'lo')]) { + watching.applyEvent(event, watcher.context); + } + joining.applyEvent(snapshot('Hel'), joiner.context); + joining.applyEvent(token('s0', 'lo'), joiner.context); + + expect(textOf(joiner.read())).toEqual(['Hello']); + expect(shape(joiner.read())).toEqual(shape(watcher.read())); + + watching.applyEvent(final('Hello'), watcher.context); + joining.applyEvent(final('Hello'), joiner.context); + expect(shape(joiner.read())).toEqual(shape(watcher.read())); + }); + + it('takes the snapshot over a stale copy of the same response', () => { + const stale: TMessage = { + messageId: 'resp-1', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + content: [{ type: ContentTypes.TEXT, text: 'H' }], + }; + const { applyEvent } = createEventApplier(); + const { context, read } = createList([user, stale]); + + applyEvent(snapshot('Hello'), context); + + expect(read()).toHaveLength(2); + expect(textOf(read())).toEqual(['Hello']); + }); + + it('keeps loaded content when the snapshot carries none', () => { + const loaded: TMessage = { + messageId: 'resp-1', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + content: [{ type: ContentTypes.TEXT, text: 'Loaded' }], + }; + const { applyEvent } = createEventApplier(); + const { context, read } = createList([user, loaded]); + + applyEvent(snapshot(''), context); + + expect(textOf(read())).toEqual(['Loaded']); + }); + + it('applies the steps a snapshot replays and skips the frames that are not message content', () => { + const { applyEvent } = createEventApplier(); + const { context, read } = createList(); + const withGap = snapshot('Hel', { replayEvents: [delta('s0', 'lo')] }) as Extract< + ChatEvent, + { type: 'sync' } + >; + withGap.data.pendingEvents = [ + { event: 'title', data: { title: 'Named' } }, + { event: 'on_token_usage', data: {} }, + delta('s0', '!'), + ] as never; + + applyEvent(withGap, context); + + expect(textOf(read())).toEqual(['Hello!']); + }); + + it('rebuilds from a second snapshot without repeating a row or a token', () => { + const { applyEvent } = createEventApplier(); + const { context, read } = createList(); + + applyEvent(snapshot('Hel'), context); + applyEvent(token('s0', 'lo'), context); + applyEvent({ type: 'error', status: 0 }, context); + applyEvent(snapshot('Hello wor'), context); + applyEvent(token('s0', 'ld'), context); + + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-1']); + expect(textOf(read())).toEqual(['Hello world']); + }); + + it('holds steps that arrive before the turn is named, then applies them in order', () => { + const { applyEvent } = createEventApplier(); + const { context, writes, read } = createList([user]); + + applyEvent(opened('s0', 0), context); + applyEvent(token('s0', 'Hi'), context); + expect(writes).toHaveLength(0); + + applyEvent(created, context); + + expect(textOf(read())).toEqual(['Hi']); + }); + + it('replaces the optimistic reply instead of adding a second response', () => { + const optimistic: TMessage = { + messageId: 'user-1_', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + }; + const { applyEvent } = createEventApplier(); + const { context, read } = createList([user, optimistic]); + + applyEvent(created, context); + applyEvent(opened('s0', 0), context); + applyEvent(token('s0', 'Hi'), context); + + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-1']); + expect(textOf(read())).toEqual(['Hi']); + }); + + it('starts a regeneration beside the saved answer instead of carrying on from it', () => { + const saved: TMessage = { + messageId: 'resp-0', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + content: [{ type: ContentTypes.TEXT, text: 'Earlier answer' }], + }; + const { applyEvent } = createEventApplier(); + const { context, read } = createList([user, saved]); + + applyEvent(created, context); + applyEvent(opened('s0', 0), context); + applyEvent(token('s0', 'Hi'), context); + + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-0', 'resp-1']); + expect(textOf(read())).toEqual(['Hi']); + expect(read()[1]).toBe(saved); + expect(read()[2].parentMessageId).toBe('user-1'); + }); + + it('leaves the leaf a compaction names as it is and starts the summary under it', () => { + const leaf: TMessage = { + messageId: 'resp-0', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + content: [{ type: ContentTypes.TEXT, text: 'Earlier answer' }], + }; + const anchor: ChatEvent = { + type: 'created', + data: { + created: true, + message: { + messageId: 'resp-0', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + }, + }, + }; + const { applyEvent } = createEventApplier(); + const { context, read } = createList([user, leaf]); + + applyEvent(anchor, context); + expect(read()).toEqual([user, leaf]); + expect(read()[1]).toBe(leaf); + + applyEvent(opened('s0', 0), context); + applyEvent(token('s0', 'Summary'), context); + + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-0', 'resp-1']); + expect(read()[1]).toBe(leaf); + expect(read()[2]).toMatchObject({ parentMessageId: 'resp-0', isCreatedByUser: false }); + expect(textOf(read())).toEqual(['Summary']); + }); + + it('keeps the text of a user leaf a compaction names', () => { + const anchor: ChatEvent = { + type: 'created', + data: { created: true, message: { ...user, text: '' } }, + }; + const { applyEvent } = createEventApplier(); + const { context, read } = createList([user]); + + applyEvent(anchor, context); + applyEvent(opened('s0', 0), context); + applyEvent(token('s0', 'Summary'), context); + + expect(read()[0]).toBe(user); + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-1']); + expect(read()[1].parentMessageId).toBe('user-1'); + }); + + it('settles a compaction on a user leaf without blanking the leaf', () => { + const anchor = { ...user, text: '' }; + const settled: ChatEvent = { + type: 'final', + data: { + final: true, + conversation: { conversationId: 'thread-1' }, + requestMessage: anchor, + responseMessage: { + messageId: 'resp-1', + conversationId: 'thread-1', + parentMessageId: 'user-1', + isCreatedByUser: false, + text: '', + content: [{ type: ContentTypes.TEXT, text: 'Summary, done' }], + }, + }, + } as ChatEvent; + const watched = createEventApplier(); + const first = createList([user]); + watched.applyEvent( + { type: 'created', data: { created: true, message: anchor } }, + first.context, + ); + watched.applyEvent(opened('s0', 0), first.context); + watched.applyEvent(token('s0', 'Summary'), first.context); + watched.applyEvent(settled, first.context); + + expect(first.read()[0]).toBe(user); + expect(textOf(first.read())).toEqual(['Summary, done']); + + /** A chat that saw only the final frame recognizes the anchor from the frame itself. */ + const late = createEventApplier(); + const second = createList([user]); + late.applyEvent(settled, second.context); + + expect(second.read()[0]).toBe(user); + expect(textOf(second.read())).toEqual(['Summary, done']); + }); + + it('keeps the user turn a regeneration snapshot placed on an empty list', () => { + const { applyEvent } = createEventApplier(); + const { context, read } = createList(); + + applyEvent(snapshot('Hel', { isRegenerate: true }), context); + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-1']); + + applyEvent(token('s0', 'lo'), context); + + expect(read().map((message) => message.messageId)).toEqual(['user-1', 'resp-1']); + expect(textOf(read())).toEqual(['Hello']); + }); + + it('settles on the final frame and starts the next turn clean', () => { + const { applyEvent } = createEventApplier(); + const { context, read } = createList(); + + applyEvent(snapshot('Hel'), context); + applyEvent(final('Hello, done'), context); + expect(textOf(read())).toEqual(['Hello, done']); + + applyEvent(token('s0', 'late'), context); + expect(textOf(read())).toEqual(['Hello, done']); + }); + + describe('with a scheduler', () => { + const scheduler = () => { + const queue: Array<() => void> = []; + const cancelled = jest.fn(); + return { + cancelled, + schedule: (flush: () => void) => { + queue.push(flush); + return cancelled; + }, + run: () => queue.splice(0).forEach((flush) => flush()), + scheduled: () => queue.length, + }; + }; + + it('writes a burst of tokens once, when the scheduled flush runs', () => { + const frame = scheduler(); + const { applyEvent } = createEventApplier({ schedule: frame.schedule }); + const { context, writes, read } = createList(); + applyEvent(snapshot(''), context); + const before = writes.length; + + applyEvent(token('s0', 'a'), context); + applyEvent(token('s0', 'b'), context); + applyEvent(token('s0', 'c'), context); + + expect(writes).toHaveLength(before); + expect(frame.scheduled()).toBe(1); + frame.run(); + expect(writes).toHaveLength(before + 1); + expect(textOf(read())).toEqual(['abc']); + }); + + it('never lets a held copy of the stream land on top of the final frame', () => { + const frame = scheduler(); + const { applyEvent } = createEventApplier({ schedule: frame.schedule }); + const { context, read } = createList(); + applyEvent(snapshot(''), context); + applyEvent(token('s0', 'partial'), context); + + applyEvent(final('whole answer'), context); + frame.run(); + + expect(frame.cancelled).toHaveBeenCalledTimes(1); + expect(textOf(read())).toEqual(['whole answer']); + }); + + it.each([{ type: 'abort' }, { type: 'error', status: 500 }])( + 'writes the held tokens at once when the turn stops ($type)', + (stop) => { + const frame = scheduler(); + const { applyEvent } = createEventApplier({ schedule: frame.schedule }); + const { context, read } = createList(); + applyEvent(snapshot(''), context); + applyEvent(token('s0', 'kept'), context); + + applyEvent(stop, context); + + expect(textOf(read())).toEqual(['kept']); + }, + ); + + it('drops a scheduled write on reset', () => { + const frame = scheduler(); + const applier = createEventApplier({ schedule: frame.schedule }); + const { context, read } = createList(); + applier.applyEvent(snapshot(''), context); + applier.applyEvent(token('s0', 'gone'), context); + + applier.reset(); + frame.run(); + + expect(textOf(read())).toEqual([]); + }); + }); + + it('reports live status a step carries beside the list', () => { + const onEffect = jest.fn(); + const { applyEvent } = createEventApplier({ onEffect }); + const { context } = createList(); + applyEvent(snapshot(''), context); + + applyEvent( + { + type: 'step', + data: { event: StepEvents.ON_SANDBOX_STARTING, data: { tool_call_id: 'call-1' } }, + } as ChatEvent, + context, + ); + + expect(onEffect).toHaveBeenCalledWith({ type: 'sandbox-starting', toolCallId: 'call-1' }); + }); +}); + +describe('a Chat that applies events the way main chat does', () => { + const createState = (initial: TMessage[]): ChatState => { + let messages = initial; + let status: ChatStatus = 'ready'; + return { + error: undefined, + connection: 'idle', + get status() { + return status; + }, + set status(next: ChatStatus) { + status = next; + }, + get messages() { + return messages; + }, + replaceMessages(next) { + messages = next; + }, + }; + }; + + it('follows a generation it joined late to the same messages, through the real transport', async () => { + const attachments: Array<{ request: ChatStreamRequest; options: ChatTransportOptions }> = []; + const wire: WireTransport = { + send: () => undefined, + reconnectToStream(request, options) { + attachments.push({ request, options }); + return { closed: false }; + }, + }; + const state = createState([]); + const chat = new Chat({ + id: 'thread-1', + state, + transport: createGenerationTransport({ + wire: () => wire, + getStatus: async () => ({ active: true, streamId: 'thread-1', createdAt: 2000 }), + }), + prepareRequest: () => { + throw new Error('view only'); + }, + applyEvent: createEventApplier().applyEvent, + }); + const flush = () => new Promise((resolve) => setImmediate(resolve)); + + const resuming = chat.resumeStream(); + await flush(); + const emit = attachments[0].options.onEvent; + emit({ type: 'open' }); + emit(snapshot('Hel')); + emit(token('s0', 'lo')); + await flush(); + + expect(chat.status).toBe('streaming'); + expect(textOf(chat.messages)).toEqual(['Hello']); + + emit(final('Hello')); + await resuming; + + expect(chat.status).toBe('ready'); + expect(chat.messages.map((message) => message.messageId)).toEqual(['user-1', 'resp-1']); + expect(textOf(chat.messages)).toEqual(['Hello']); + chat.dispose(); + }); +}); diff --git a/packages/chat/src/core/apply.ts b/packages/chat/src/core/apply.ts new file mode 100644 index 00000000000..14ca7ba4ae5 --- /dev/null +++ b/packages/chat/src/core/apply.ts @@ -0,0 +1,304 @@ +import { StepEvents } from 'librechat-data-provider'; +import type { + TMessage, + ChatEvent, + ChatFrame, + ChatSyncFrame, + ChatFinalFrame, + EventSubmission, + ChatCreatedFrame, +} from 'librechat-data-provider'; +import type { StepEvent, StepEffect, StepContext } from './reducer'; +import type { ChatEventContext } from './chat'; +import { + reduceStep, + flushSummary, + flushResponses, + resetStreamState, + syncStreamMessage, + createStreamState, +} from './reducer'; +import { buildResumeEventSubmission, buildSubmissionFromResumeState } from './resumable/resume'; +import { buildCreatedInitialResponse } from './resumable/messages'; +import { placeResumedResponse } from './resumable/placement'; +import { normalizeFrame } from './transport/frames'; +import { mergeFinalFrame } from './chat'; + +/** A step's side channel that is not a write to the message list. */ +export type EventApplierEffect = Exclude; + +export type EventApplierInit = { + /** + * Defers the write of streamed tokens and returns how to cancel it, so a burst of deltas costs + * one write. A browser passes `requestAnimationFrame`. Without it every delta writes at once. + */ + schedule?: (flush: () => void) => () => void; + /** Live status a step reports beside the list: sandbox start, nested subagents, announcements. */ + onEffect?: (effect: EventApplierEffect) => void; +}; + +export type EventApplier = { + /** `ChatInit.applyEvent`: folds a turn's `created`, `sync`, `step` and `final` frames in. */ + applyEvent: (event: ChatEvent, context: ChatEventContext) => void; + /** Writes the streamed tokens a scheduled flush still holds. */ + flush: () => void; + /** Forgets the turn being applied and drops any scheduled write. */ + reset: () => void; +}; + +type HeldFlush = { responses: Set; summaries: Set; messages: TMessage[] }; + +const stepContext = (submission: EventSubmission, context: ChatEventContext): StepContext => ({ + submission, + getMessages: () => context.messages, + setMessages: (messages) => context.write(messages), +}); + +/** + * Whether a frame's message is the identity of a row the list holds, not a turn's content: a + * compaction names the leaf it summarizes up to, whoever wrote it, and sends no text for it. + */ +const isAnchor = (held: TMessage, named: Partial): boolean => + held.isCreatedByUser !== true || !named.text; + +/** + * A final frame as its turn settles it. A turn that made no user row still names one in its + * request: a regeneration its saved turn, a compaction its anchor. Merging that onto the row + * would blank what it holds, so such a turn settles its response alone, as main chat's final + * handler does. A chat that saw only the final frame recognizes the anchor from the frame. + */ +function settledFinal( + data: ChatFinalFrame, + turn: EventSubmission | undefined, + messages: TMessage[], +): ChatFinalFrame { + const request = data.requestMessage; + if (request?.messageId == null) { + return data; + } + const held = messages.find((message) => message.messageId === request.messageId); + const reused = turn?.isRegenerate === true || (held != null && isAnchor(held, request)); + return reused ? { ...data, requestMessage: null } : data; +} + +/** + * The turn a `created` frame names: its user message and the reply the turn starts from. + * + * A row the list already holds is never rewritten from the frame. A regeneration reuses a saved + * user turn, and a compaction makes no turn at all: its frame names the leaf it summarizes up to, + * with that leaf's author and no text. Only a message the list does not hold is a new turn to add. + * + * Sharing the user message as a parent does not make a row this turn's reply: a regeneration's + * earlier answers are its children too. Only a row under the id the frame names, or under the + * padded id a turn draws its reply at before the server names it, is the reply to carry on from. + * Any other reply already under the message is a saved answer, so the turn starts empty beside + * it and leaves it in place, as it does under a compaction's anchor. + */ +function createdSubmission(data: ChatCreatedFrame, messages: TMessage[]): EventSubmission { + const loaded = messages.find((message) => message.messageId === data.message.messageId); + const anchored = + loaded == null ? data.message.isCreatedByUser === false : isAnchor(loaded, data.message); + const userMessage = + loaded ?? + ({ ...data.message, isCreatedByUser: data.message.isCreatedByUser !== false } as TMessage); + const optimisticId = `${userMessage.messageId}_`; + let placeholder: TMessage | undefined; + let answered = false; + for (const message of messages) { + if (message.isCreatedByUser || message.parentMessageId !== userMessage.messageId) { + continue; + } + if (message.messageId === optimisticId || message.messageId === userMessage.responseMessageId) { + placeholder = message; + } else { + answered = true; + } + } + const initialResponse = + placeholder ?? + buildCreatedInitialResponse({ + initialResponse: { text: '', isCreatedByUser: false } as TMessage, + userMessage, + }); + return { + conversation: { conversationId: userMessage.conversationId }, + endpointOption: {}, + isRegenerate: anchored || answered, + ...(anchored && { compact: true }), + messages: loaded != null || anchored ? messages : [...messages, userMessage], + userMessage, + initialResponse, + } as EventSubmission; +} + +/** + * The event application main chat runs, as a `Chat` reducer: the step reducer for a turn's + * graph events, and a resume snapshot rebuilt and placed by the same code a reattached stream + * uses there. A `Chat` constructed with it holds, for the same frames, the messages main chat + * holds. + * + * Stateful, one per `Chat`: it keeps the streaming copy of each response between events. Frames + * that describe something other than message content (usage, titles, attachments, steers, labels + * and pauses) are left to `onEvent`. + */ +export function createEventApplier(init: EventApplierInit = {}): EventApplier { + const stream = createStreamState(); + let submission: EventSubmission | undefined; + let context: ChatEventContext | undefined; + /** Steps that arrived before a frame named the turn they belong to. */ + let early: StepEvent[] = []; + let held: HeldFlush | undefined; + let cancel: (() => void) | undefined; + + const drop = (): void => { + cancel?.(); + cancel = undefined; + held = undefined; + }; + + const flush = (): void => { + const pending = held; + drop(); + if (pending == null || submission == null || context == null) { + return; + } + const ctx = stepContext(submission, context); + flushResponses(stream, ctx, pending.responses, pending.messages); + for (const messageId of pending.summaries) { + flushSummary(stream, ctx, messageId, pending.messages); + } + }; + + const reset = (): void => { + drop(); + resetStreamState(stream); + submission = undefined; + early = []; + }; + + const settle = (effects: StepEffect[]): void => { + for (const effect of effects) { + if (effect.type !== 'flush' && effect.type !== 'summary') { + init.onEffect?.(effect); + continue; + } + held ??= { responses: new Set(), summaries: new Set(), messages: effect.messages }; + held.messages = effect.messages; + (effect.type === 'flush' ? held.responses : held.summaries).add(effect.messageId); + } + if (held != null && init.schedule != null) { + cancel ??= init.schedule(flush); + } + }; + + const step = (event: StepEvent): void => { + if (submission == null || context == null) { + early.push(event); + return; + } + settle(reduceStep(stream, event, stepContext(submission, context))); + }; + + /** Frames a snapshot carries beside its content; only the graph's step events write messages. */ + const replay = (frames: ReadonlyArray | undefined): void => { + for (const frame of frames ?? []) { + const event = normalizeFrame(frame as ChatFrame); + if (event?.type === 'step') { + step(event.data as StepEvent); + } + } + }; + + const created = (data: ChatCreatedFrame, ctx: ChatEventContext): void => { + if (!data.message?.messageId) { + return; + } + const waiting = early; + reset(); + const next = createdSubmission(data, ctx.messages); + /** The list with the turn's message in it, which a regeneration's steps build on. */ + if (next.messages !== ctx.messages) { + ctx.write(next.messages); + } + submission = next; + for (const event of waiting) { + step(event); + } + }; + + const sync = (data: ChatSyncFrame, ctx: ChatEventContext): void => { + const { resumeState } = data; + if (resumeState == null) { + replay(data.pendingEvents); + return; + } + /** What a scheduled write holds predates the snapshot, and so do steps not yet placed. */ + drop(); + early = []; + const conversationId = + resumeState.conversationId ?? + resumeState.userMessage?.conversationId ?? + data.conversationId ?? + submission?.conversation?.conversationId ?? + ''; + const previous = + submission ?? + (buildSubmissionFromResumeState( + resumeState, + conversationId, + ctx.messages, + conversationId, + ) as EventSubmission); + const next = buildResumeEventSubmission(previous, previous.userMessage, resumeState); + submission = next; + + for (const runStep of resumeState.runSteps ?? []) { + step({ event: StepEvents.ON_RUN_STEP, data: runStep }); + } + + if (resumeState.aggregatedContent && next.userMessage?.messageId) { + const placement = placeResumedResponse({ + messages: ctx.messages, + submission: next, + userMessage: next.userMessage, + resumeState, + previous, + }); + /** The snapshot's content is indexed from zero, so later steps stop offsetting by an edit. */ + if (!placement.preservedLoadedContent) { + next.editPrefixCleared = true; + } + ctx.write(placement.messages); + /** A regeneration's steps rebuild the list from this baseline, so it holds the placed turn. */ + next.messages = placement.messages; + settle(syncStreamMessage(stream, placement.response)); + } + + replay(resumeState.replayEvents); + replay(data.pendingEvents); + }; + + const applyEvent = (event: ChatEvent, ctx: ChatEventContext): void => { + context = ctx; + if (event.type === 'created') { + created(event.data, ctx); + } else if (event.type === 'sync') { + sync(event.data, ctx); + } else if (event.type === 'step') { + step(event.data as StepEvent); + } else if (event.type === 'final') { + /** The final frame is authoritative; a held copy of the stream must not land on top of it. */ + drop(); + mergeFinalFrame(settledFinal(event.data, submission, ctx.messages), ctx); + reset(); + } else if (event.type === 'abort' || event.type === 'error') { + /** A turn that stops here keeps every token it streamed. */ + flush(); + } + if (init.schedule == null) { + flush(); + } + }; + + return { applyEvent, flush, reset }; +} diff --git a/packages/chat/src/core/chat.ts b/packages/chat/src/core/chat.ts index 79c2635de87..3debb743ed7 100644 --- a/packages/chat/src/core/chat.ts +++ b/packages/chat/src/core/chat.ts @@ -82,6 +82,7 @@ export type ChatPreparedRequest = { /** The message list as an event handler reads and writes it. */ export type ChatEventContext = { + /** The list as it stands when read, so a write made earlier in the same event is seen. */ readonly messages: TMessage[]; write(messages: TMessage[]): void; }; @@ -609,7 +610,8 @@ const mergeById = ( return next; }; -const mergeFinal = (data: ChatFinalFrame, context: ChatEventContext) => { +/** Merges a final frame's request and response onto the rows they name; what `Chat` does alone. */ +export const mergeFinalFrame = (data: ChatFinalFrame, context: ChatEventContext): void => { const next = mergeById(context.messages, [data.requestMessage, data.responseMessage]); if (next) { context.write(next); @@ -618,10 +620,20 @@ const mergeFinal = (data: ChatFinalFrame, context: ChatEventContext) => { const defaultApplyEvent = (event: ChatEvent, context: ChatEventContext) => { if (event.type === 'final') { - mergeFinal(event.data, context); + mergeFinalFrame(event.data, context); } }; +const createEventContext = ( + read: () => TMessage[], + write: (messages: TMessage[]) => void, +): ChatEventContext => ({ + get messages() { + return read(); + }, + write, +}); + /** A tool call that may hold a subagent's own tool calls, where a paused tool can live. */ type NestedToolCall = Agents.ToolCall & { subagent_content?: TMessageContentParts[] }; @@ -833,6 +845,10 @@ export class Chat { readonly id: string; private readonly init: ChatInit; private readonly executor = new SerialJobExecutor(); + private readonly eventContext = createEventContext( + () => this.messages, + (messages) => this.write(messages), + ); private readonly listeners = new Set<() => void>(); private releaseState?: () => void; private active?: ActiveRequest; @@ -2365,10 +2381,7 @@ export class Chat { this.reconcilePause(event, older, disconnect); if (folded && !this.disposed) { const applyEvent = this.init.applyEvent ?? defaultApplyEvent; - applyEvent(folded, { - messages: this.messages, - write: (messages) => this.write(messages), - }); + applyEvent(folded, this.eventContext); } /** A listener told of that write may have disposed the chat, which then owns no more state. */ if (this.disposed) { diff --git a/packages/chat/src/core/index.ts b/packages/chat/src/core/index.ts index 9f1a20b6806..f9cc1f29efa 100644 --- a/packages/chat/src/core/index.ts +++ b/packages/chat/src/core/index.ts @@ -3,6 +3,7 @@ export * from './pending'; export * from './decision'; export * from './resumable'; export * from './chat'; +export * from './apply'; export * from './steps'; export * from './steer'; export * from './types'; diff --git a/packages/chat/src/core/resumable/placement.ts b/packages/chat/src/core/resumable/placement.ts index 1619e7cabbf..5fa79998c5a 100644 --- a/packages/chat/src/core/resumable/placement.ts +++ b/packages/chat/src/core/resumable/placement.ts @@ -1,4 +1,5 @@ -import type { Agents, TMessage, TSubmission } from 'librechat-data-provider'; +import type { Agents, TMessage, TSubmission, EventSubmission } from 'librechat-data-provider'; +import { preferDefinedString } from './messages'; export type ResumeMessageIndexes = { userIndex: number; @@ -140,6 +141,104 @@ export const mergeResumeMessages = ( return [...nextMessages, userMessage, responseMessage]; }; +/** Where a resume snapshot's response landed in the list, and what the list is now. */ +export type ResumedResponsePlacement = { + messages: TMessage[]; + response: TMessage; + /** The index the response already held, under its durable id or its preliminary one; else -1. */ + responseIndex: number; + /** The list's own content was kept because the snapshot carried none. */ + preservedLoadedContent: boolean; +}; + +/** + * Rebuilds the response a resume snapshot describes and places it, with its user turn, in the + * list. The snapshot's aggregated content is authoritative over what the list holds, with one + * exception: an EMPTY snapshot never erases content already loaded for the same response, which + * would leave a bare cursor. + * + * When loaded content is not preserved, the response holds completion-local content indexed from + * zero, so the caller must stop offsetting later steps by an edit's retained prefix. + */ +export function placeResumedResponse({ + messages, + submission, + userMessage, + resumeState, + previous, +}: { + messages: TMessage[]; + /** The submission already rebuilt from the snapshot (`buildResumeEventSubmission`). */ + submission: EventSubmission; + userMessage: TMessage; + resumeState: Agents.ResumeState; + /** + * The submission the run held before the snapshot. Its optimistic rows are replaced in place + * when the snapshot assigns the turn its durable ids. + */ + previous?: Pick | null; +}): ResumedResponsePlacement { + const { initialResponse } = submission; + const preliminaryResponseMessageId = previous?.initialResponse?.messageId; + const previousUserMessageId = previous?.userMessage?.messageId; + const preliminaryUserMessageId = + previousUserMessageId && + (previous?.initialResponse?.parentMessageId === previousUserMessageId || + preliminaryResponseMessageId === `${previousUserMessageId}_`) + ? previousUserMessageId + : undefined; + const responseId = initialResponse.messageId; + const resumedContent = initialResponse.content; + const indexes = getResumeMessageIndexes( + messages, + userMessage.messageId, + responseId, + preliminaryUserMessageId, + preliminaryResponseMessageId, + ); + const responseIndex = + indexes.responseIndex >= 0 ? indexes.responseIndex : indexes.preliminaryResponseIndex; + const anchored = submission.compact === true; + + if (responseIndex < 0) { + const response = { + ...initialResponse, + messageId: responseId, + parentMessageId: userMessage.messageId, + content: resumedContent, + isCreatedByUser: false, + } as TMessage; + return { + messages: mergeResumeMessages(messages, userMessage, response, indexes, anchored), + response, + responseIndex, + preservedLoadedContent: false, + }; + } + + const loaded = messages[responseIndex]; + const loadedContent = loaded?.content; + /** Require an existing content array so it is never swapped for `undefined`. */ + const preservedLoadedContent = + (resumedContent?.length ?? 0) === 0 && Array.isArray(loadedContent) && loadedContent.length > 0; + const response = { + ...initialResponse, + ...loaded, + messageId: responseId, + parentMessageId: userMessage.messageId, + content: preservedLoadedContent ? loadedContent : resumedContent, + sender: loaded?.sender ?? initialResponse.sender, + iconURL: preferDefinedString(loaded?.iconURL, initialResponse.iconURL ?? resumeState.iconURL), + model: preferDefinedString(loaded?.model, initialResponse.model ?? resumeState.model), + } as TMessage; + return { + messages: mergeResumeMessages(messages, userMessage, response, indexes, anchored), + response, + responseIndex, + preservedLoadedContent, + }; +} + export function hasSubmissionUserMessage( submission: TSubmission | null, messages: TMessage[] | undefined, diff --git a/packages/chat/src/core/resumable/sync.ts b/packages/chat/src/core/resumable/sync.ts index 578ba515867..e80a735d483 100644 --- a/packages/chat/src/core/resumable/sync.ts +++ b/packages/chat/src/core/resumable/sync.ts @@ -8,7 +8,6 @@ import { } from 'librechat-data-provider'; import type { Agents, - TMessage, ChatSyncFrame, TPendingSteer, TTokenUsageEvent, @@ -20,10 +19,9 @@ import type { } from 'librechat-data-provider'; import type { StreamContext } from './terminal'; import type { StepEvent } from '../reducer'; -import { getResumeMessageIndexes, mergeResumeMessages } from './placement'; import { getStepEventId, isOAuthStepEvent } from './events'; import { buildResumeEventSubmission } from './resume'; -import { preferDefinedString } from './messages'; +import { placeResumedResponse } from './placement'; /** * A resume's snapshot of the run: adopts the server's identity for the turn, rebuilds the @@ -40,16 +38,9 @@ export function handleSync(ctx: StreamContext, data: ChatSyncFrame): void { }); host.pane.renewActiveRun(); - /** Keep the current run's preliminary id long enough to replace that optimistic - * row in place if this snapshot assigns its durable response id. */ - const preliminaryResponseMessageId = sub.submission.initialResponse?.messageId; - const currentUserMessageId = sub.submission.userMessage?.messageId; - const preliminaryUserMessageId = - currentUserMessageId && - (sub.submission.initialResponse?.parentMessageId === currentUserMessageId || - preliminaryResponseMessageId === `${currentUserMessageId}_`) - ? currentUserMessageId - : undefined; + /** Kept so the snapshot can replace this run's optimistic rows in place if it assigns the + * turn its durable ids. */ + const previous = sub.submission; const resumeSubmission = buildResumeEventSubmission( sub.submission, sub.userMessage, @@ -98,112 +89,43 @@ export function handleSync(ctx: StreamContext, data: ChatSyncFrame): void { if (data.resumeState?.aggregatedContent && sub.userMessage?.messageId) { const messages = host.messages.get() ?? []; - const userMsgId = sub.userMessage.messageId; - const resumedContent = resumeSubmission.initialResponse.content; - const hasResumedContent = (resumedContent?.length ?? 0) > 0; - const responseId = resumeSubmission.initialResponse.messageId; - const messageIndexes = getResumeMessageIndexes( + const placement = placeResumedResponse({ messages, - userMsgId, - responseId, - preliminaryUserMessageId, - preliminaryResponseMessageId, - ); - const responseIdx = - messageIndexes.responseIndex >= 0 - ? messageIndexes.responseIndex - : messageIndexes.preliminaryResponseIndex; + submission: resumeSubmission, + userMessage: sub.userMessage, + resumeState: data.resumeState, + previous, + }); + const { response, responseIndex, preservedLoadedContent } = placement; host.logger.log('ResumableSSE', 'SYNC update', { - userMsgId, - responseId, - responseIdx, - foundMessageId: responseIdx >= 0 ? messages[responseIdx]?.messageId : null, + userMsgId: sub.userMessage.messageId, + responseId: response.messageId, + responseIdx: responseIndex, + foundMessageId: responseIndex >= 0 ? messages[responseIndex]?.messageId : null, messagesCount: messages.length, aggregatedContentLength: data.resumeState.aggregatedContent?.length, + preservedExistingContent: preservedLoadedContent, }); - if (responseIdx >= 0) { - const oldContent = messages[responseIdx]?.content; - /** An EMPTY resume snapshot is not authoritative over content we already loaded - * for the SAME generation-owned response: assigning it would erase that content - * and leave a bare cursor. Require an existing content array so it is never - * swapped for `undefined`. */ - const preserveLoadedContent = - !hasResumedContent && Array.isArray(oldContent) && oldContent.length > 0; - /** - * Replacing the response with `aggregatedContent` drops the - * prefix an edited resubmission had retained: the snapshot is - * completion-local, indexed from zero. Every later event must - * therefore stop offsetting, or it writes past the end of the - * shorter array: for an activity label that means the fill - * misses its own reservation and the placeholder never - * resolves. Preserving `oldContent` keeps the prefix, and with - * it the offset. Run steps and labels both read this. - */ - if (!preserveLoadedContent) { - run.editPrefixCleared = true; - } - const responseMessage = { - ...resumeSubmission.initialResponse, - ...messages[responseIdx], - messageId: responseId, - parentMessageId: userMsgId, - content: preserveLoadedContent ? oldContent : resumedContent, - sender: messages[responseIdx]?.sender ?? resumeSubmission.initialResponse.sender, - iconURL: preferDefinedString( - messages[responseIdx]?.iconURL, - resumeSubmission.initialResponse.iconURL ?? data.resumeState.iconURL, - ), - model: preferDefinedString( - messages[responseIdx]?.model, - resumeSubmission.initialResponse.model ?? data.resumeState.model, - ), - } as TMessage; - const updated = mergeResumeMessages( - messages, - sub.userMessage, - responseMessage, - messageIndexes, - resumeSubmission.compact === true, - ); - host.logger.log('ResumableSSE', 'SYNC updating message', { - messageId: responseMessage.messageId, - oldContentLength: Array.isArray(oldContent) ? oldContent.length : 0, - newContentLength: data.resumeState.aggregatedContent?.length, - preservedExistingContent: preserveLoadedContent, - }); - host.messages.set(updated); - host.events.resetContentHandler(); - host.events.syncStepMessage(responseMessage); - host.logger.log('ResumableSSE', 'SYNC complete, handlers synced'); - } else { - /** Same reasoning as the matched branch above: this row is - * built straight from the server's completion-local - * `aggregatedContent`, so it holds no retained prefix and - * later steps and labels must stop offsetting. Setting it - * only in the matched branch left this path adding an offset - * to indices that were already absolute. */ + /** + * Replacing the response with `aggregatedContent` drops the + * prefix an edited resubmission had retained: the snapshot is + * completion-local, indexed from zero. Every later event must + * therefore stop offsetting, or it writes past the end of the + * shorter array: for an activity label that means the fill + * misses its own reservation and the placeholder never + * resolves. Preserving the loaded content keeps the prefix, and + * with it the offset. Run steps and labels both read this. A row + * built fresh from the snapshot holds no retained prefix either. + */ + if (!preservedLoadedContent) { run.editPrefixCleared = true; - const newMessage = { - ...resumeSubmission.initialResponse, - messageId: responseId, - parentMessageId: userMsgId, - content: resumedContent, - isCreatedByUser: false, - } as TMessage; - host.messages.set( - mergeResumeMessages( - messages, - sub.userMessage, - newMessage, - messageIndexes, - resumeSubmission.compact === true, - ), - ); - host.events.resetContentHandler(); - host.events.syncStepMessage(newMessage); } + host.messages.set(placement.messages); + host.events.resetContentHandler(); + host.events.syncStepMessage(response); + host.logger.log('ResumableSSE', 'SYNC complete, handlers synced'); } for (const prompt of pendingOAuthPrompts) {