diff --git a/api/server/routes/agents/__tests__/abort.spec.js b/api/server/routes/agents/__tests__/abort.spec.js index a980ec60fce..aee548279c2 100644 --- a/api/server/routes/agents/__tests__/abort.spec.js +++ b/api/server/routes/agents/__tests__/abort.spec.js @@ -181,6 +181,41 @@ describe('Agent Abort Endpoint', () => { ); }); + it('should refuse to abort a background child job, which its task store owns', async () => { + const jobStreamId = 'child-thread-123'; + + mockGenerationJobManager.getJob.mockResolvedValue({ + metadata: { + userId: 'test-user-123', + subagentTaskId: 'task-1', + subagentParentConversationId: 'parent-1', + }, + }); + + const response = await request(app) + .post('/api/agents/chat/abort') + .send({ conversationId: jobStreamId }); + + expect(response.status).toBe(409); + expect(response.body).toMatchObject({ code: 'CHILD_THREAD_READ_ONLY' }); + expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled(); + expect(mockSaveMessage).not.toHaveBeenCalled(); + expect(mockSaveConvo).not.toHaveBeenCalled(); + }); + + it("should answer 403, not 409, for another user's background child job", async () => { + mockGenerationJobManager.getJob.mockResolvedValue({ + metadata: { userId: 'other-user-456', subagentTaskId: 'task-1' }, + }); + + const response = await request(app) + .post('/api/agents/chat/abort') + .send({ conversationId: 'child-thread-123' }); + + expect(response.status).toBe(403); + expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled(); + }); + it('should fail closed when job has no userId metadata', async () => { const jobStreamId = 'test-stream-123'; diff --git a/api/server/routes/agents/__tests__/idempotencyLimiter.spec.js b/api/server/routes/agents/__tests__/idempotencyLimiter.spec.js index dcf0da92599..c0c4da51eb4 100644 --- a/api/server/routes/agents/__tests__/idempotencyLimiter.spec.js +++ b/api/server/routes/agents/__tests__/idempotencyLimiter.spec.js @@ -33,6 +33,7 @@ jest.mock('@librechat/data-schemas', () => ({ jest.mock('@librechat/api', () => ({ isEnabled: jest.fn(() => true), + createBackgroundChildAbortGuard: jest.fn(() => (_req, _res, next) => next()), detectGenerationRetry: async (req, _res, next) => { req._isConfirmedGenerationRetry = await mockHasGenerationClaim( req.user?.id, diff --git a/api/server/routes/agents/__tests__/streamTenant.spec.js b/api/server/routes/agents/__tests__/streamTenant.spec.js index e96b74711a9..6c9231234bc 100644 --- a/api/server/routes/agents/__tests__/streamTenant.spec.js +++ b/api/server/routes/agents/__tests__/streamTenant.spec.js @@ -959,12 +959,18 @@ describe('SSE stream tenant isolation', () => { }); it('reports an already-settled generation when natural completion wins the abort CAS', async () => { + /** Read by the child-job guard, by the route, and again after the abort lost its CAS. */ mockGenerationJobManager.getJob .mockResolvedValueOnce({ metadata: { userId: 'user-123', generationProtocolVersion: 2 }, status: 'running', createdAt: 1000, }) + .mockResolvedValueOnce({ + metadata: { userId: 'user-123', generationProtocolVersion: 2 }, + status: 'running', + createdAt: 1000, + }) .mockResolvedValueOnce({ metadata: { userId: 'user-123', generationProtocolVersion: 2 }, status: 'complete', @@ -996,12 +1002,18 @@ describe('SSE stream tenant isolation', () => { }); it('keeps the legacy abort ACK shape when natural completion wins for protocol v1', async () => { + /** Read by the child-job guard, by the route, and again after the abort lost its CAS. */ mockGenerationJobManager.getJob .mockResolvedValueOnce({ metadata: { userId: 'user-123', generationProtocolVersion: 1 }, status: 'running', createdAt: 1000, }) + .mockResolvedValueOnce({ + metadata: { userId: 'user-123', generationProtocolVersion: 1 }, + status: 'running', + createdAt: 1000, + }) .mockResolvedValueOnce({ metadata: { userId: 'user-123', generationProtocolVersion: 1 }, status: 'complete', diff --git a/api/server/routes/agents/index.js b/api/server/routes/agents/index.js index 1962f1bc994..48dfd4a3bf4 100644 --- a/api/server/routes/agents/index.js +++ b/api/server/routes/agents/index.js @@ -3,6 +3,7 @@ const { reportLocatorTraversalFailure, isEnabled, GenerationJobManager, + createBackgroundChildAbortGuard, TERMINAL_PUBLICATION_RECONNECT_ERROR, hasPersistableAbortContent, announceStoppedReply, @@ -622,6 +623,12 @@ router.get('/chat/status/:conversationId', async (req, res) => { * @access Private * @description Mounted before chatRouter to bypass buildEndpointOption middleware */ +router.post( + '/chat/abort', + createBackgroundChildAbortGuard({ + getJob: (streamId) => GenerationJobManager.getJob(streamId), + }), +); router.post('/chat/abort', chatConfigMiddleware, async (req, res, next) => { logger.debug(`[AgentStream] ========== ABORT ENDPOINT HIT ==========`); logger.debug(`[AgentStream] Method: ${req.method}, Path: ${req.path}`); diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.js b/api/server/services/Endpoints/agents/subagentThreadStore.js index 1631f97f10e..210990f5061 100644 --- a/api/server/services/Endpoints/agents/subagentThreadStore.js +++ b/api/server/services/Endpoints/agents/subagentThreadStore.js @@ -5,6 +5,7 @@ const { duplicateIoRedisClient, createIoRedisSubscriber, createSubagentThreadTaskStore, + createChildGenerationHost, createSubagentCompletionWakeupHandler, GenerationJobManager, RedisSubagentTaskControlTransport, @@ -80,6 +81,7 @@ const subagentThreadTaskStore = createSubagentThreadTaskStore( renewOwnerAdmission: db.renewSubagentAdmission, releaseOwnerAdmission: db.releaseSubagentAdmission, cancelUnroutedTask: cancelUnroutedGeneration, + childGenerations: createChildGenerationHost(GenerationJobManager), onTaskPrepared: completionWakeupHandler, onTaskSettled: (userId, conversationId, taskIds) => expediteCompletionWakeups({ user: userId, conversationId, taskIds }), diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js index b42198f5908..08368843a80 100644 --- a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js +++ b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js @@ -7,6 +7,8 @@ const mockTaskStore = { destroyActivityStream: jest.fn(), }; const mockCompletionWakeupHandler = jest.fn().mockResolvedValue(undefined); +const mockChildGenerations = { open: jest.fn() }; +const mockGenerationJobManager = { getJob: jest.fn(), abortJob: jest.fn() }; jest.mock('@librechat/api', () => ({ cacheConfig: { USE_REDIS: true, REDIS_KEY_PREFIX: 'test:' }, @@ -15,6 +17,8 @@ jest.mock('@librechat/api', () => ({ duplicateIoRedisClient: jest.fn(), createIoRedisSubscriber: jest.fn(), createSubagentThreadTaskStore: jest.fn(() => mockTaskStore), + createChildGenerationHost: jest.fn(() => mockChildGenerations), + GenerationJobManager: mockGenerationJobManager, createSubagentCompletionWakeupHandler: jest.fn(() => mockCompletionWakeupHandler), RedisSubagentTaskControlTransport: jest.fn(), RedisEventTransport: jest.fn(), @@ -54,12 +58,14 @@ const { registerShutdownTask, duplicateIoRedisClient, createIoRedisSubscriber, + createChildGenerationHost, createSubagentThreadTaskStore, } = require('@librechat/api'); const subagentThreadTaskStore = require('./subagentThreadStore'); const { configureSubagentTaskRouting } = subagentThreadTaskStore; const taskStoreOptions = createSubagentThreadTaskStore.mock.calls[0][1]; const taskStoreMethods = createSubagentThreadTaskStore.mock.calls[0][0]; +const childGenerationHostArgs = createChildGenerationHost.mock.calls[0]; const db = require('~/models'); const activityPrepareRegistration = registerShutdownTask.mock.calls.find( ([name]) => name === 'subagent activity streams prepare', @@ -76,6 +82,11 @@ describe('subagent thread Redis lifecycle', () => { expect(taskStoreMethods.getSubagentTaskControlReplay).toBe(db.getSubagentTaskControlReplay); }); + it('gives the host store a child stream backed by the generation job manager', () => { + expect(childGenerationHostArgs).toEqual([mockGenerationJobManager]); + expect(taskStoreOptions.childGenerations).toBe(mockChildGenerations); + }); + it('pre-registers completion wakeups for every prepared task', async () => { await taskStoreOptions.onTaskPrepared({ taskId: 'task-1' }); diff --git a/packages/api/src/agents/guard.spec.ts b/packages/api/src/agents/guard.spec.ts index 4424757e3c3..71394e7c4c5 100644 --- a/packages/api/src/agents/guard.spec.ts +++ b/packages/api/src/agents/guard.spec.ts @@ -3,6 +3,7 @@ import request from 'supertest'; import type { AllMethods, IConversation } from '@librechat/data-schemas'; import { CHILD_THREAD_READ_ONLY_ERROR, + createBackgroundChildAbortGuard, createSubagentThreadTurnGuard, isSubagentThreadWriteBlocked, } from './guard'; @@ -354,3 +355,102 @@ describe('subagent child-thread write policy', () => { expect(getConvo).toHaveBeenCalledWith('owner', 'child-conversation'); }); }); + +describe('background child abort guard', () => { + const childJob = { metadata: { userId: 'owner', subagentTaskId: 'task-1' } }; + + function abortApp( + getJob: (streamId: string) => Promise, + user: { id?: string; tenantId?: string } = { id: 'owner' }, + ) { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + (req as express.Request & { user?: typeof user }).user = user; + next(); + }); + app.post( + '/abort', + createBackgroundChildAbortGuard({ + getJob: getJob as Parameters[0]['getJob'], + }), + (_req, res) => { + res.status(200).json({ reached: 'route' }); + }, + ); + return app; + } + + it("answers 409 for the owner's background child job without reaching the route", async () => { + const getJob = jest.fn(async () => childJob); + + const response = await request(abortApp(getJob)).post('/abort').send({ streamId: 'thread-1' }); + + expect(response.status).toBe(409); + expect(response.body).toEqual({ + code: 'CHILD_THREAD_READ_ONLY', + error: CHILD_THREAD_READ_ONLY_ERROR, + }); + expect(getJob).toHaveBeenCalledWith('thread-1'); + }); + + it.each([ + ['conversationId', { conversationId: 'thread-1' }], + ['abortKey', { abortKey: 'thread-1:suffix' }], + ['the first concrete id after a placeholder', { streamId: 'new', conversationId: 'thread-1' }], + ])('reads the target from %s', async (_name, body) => { + const getJob = jest.fn(async () => childJob); + + const response = await request(abortApp(getJob)).post('/abort').send(body); + + expect(response.status).toBe(409); + expect(getJob).toHaveBeenCalledWith('thread-1'); + }); + + it.each([ + ['an ordinary chat job', { metadata: { userId: 'owner' } }], + ["another user's child job", { metadata: { userId: 'someone-else', subagentTaskId: 't' } }], + [ + "another tenant's child job", + { metadata: { userId: 'owner', tenantId: 'tenant-b', subagentTaskId: 't' } }, + ], + ['a job with no owner', { metadata: { subagentTaskId: 't' } }], + ['no job', undefined], + ])('falls through to the route for %s', async (_name, job) => { + const response = await request(abortApp(async () => job, { id: 'owner', tenantId: 'tenant-a' })) + .post('/abort') + .send({ streamId: 'thread-1' }); + + expect(response.body).toEqual({ reached: 'route' }); + }); + + it('falls through without reading a job for a placeholder, a missing id or no user', async () => { + const getJob = jest.fn(async () => childJob); + + for (const body of [{ streamId: 'new' }, {}, { streamId: 42 }, { streamId: 'x'.repeat(513) }]) { + const response = await request(abortApp(getJob)).post('/abort').send(body); + expect(response.body).toEqual({ reached: 'route' }); + } + const anonymous = await request(abortApp(getJob, {})).post('/abort').send({ streamId: 't' }); + expect(anonymous.body).toEqual({ reached: 'route' }); + expect(getJob).not.toHaveBeenCalled(); + }); + + it('fails closed, without reaching the route, when the job lookup fails', async () => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(); + try { + const response = await request( + abortApp(async () => { + throw new Error('store down'); + }), + ) + .post('/abort') + .send({ streamId: 'thread-1' }); + + expect(response.status).toBe(500); + expect(response.body).not.toEqual({ reached: 'route' }); + } finally { + consoleError.mockRestore(); + } + }); +}); diff --git a/packages/api/src/agents/guard.ts b/packages/api/src/agents/guard.ts index 7e0389105af..1702bbfbf72 100644 --- a/packages/api/src/agents/guard.ts +++ b/packages/api/src/agents/guard.ts @@ -9,6 +9,93 @@ import { isAgentEventRetentionActive } from './eventRetention'; export const CHILD_THREAD_READ_ONLY_ERROR = 'This subagent thread is view-only. Continue it from its parent agent or create a separate chat.'; +export const CHILD_THREAD_READ_ONLY_CODE = 'CHILD_THREAD_READ_ONLY'; + +export type BackgroundChildJobRefusal = { + status: 409; + body: { code: typeof CHILD_THREAD_READ_ONLY_CODE; error: string }; +}; + +/** + * The answer for a chat route asked to end or steer a background child's job. The subagent + * task store owns that run: it stops and steers the child through its own control endpoint + * and writes the child's rows itself, so a chat route acting on the job would write a second, + * chat-shaped history into a view-only thread. Null for every other job. + */ +type AbortGuardJob = { + metadata?: { userId?: string; tenantId?: string; subagentTaskId?: string }; +}; + +const MAX_ABORT_TARGET_LENGTH = 512; + +/** The concrete stream id an abort request names, in the route's own order. The `new` + * placeholder names none: its fallback reads the active chat list, which holds no child. */ +function abortTargetId(body: unknown): string | null { + if (body == null || typeof body !== 'object') return null; + const { streamId, conversationId, abortKey } = body as Record; + const prefix = typeof abortKey === 'string' ? abortKey.split(':')[0] : undefined; + for (const value of [streamId, conversationId, prefix]) { + if ( + typeof value === 'string' && + value !== '' && + value !== 'new' && + value.length <= MAX_ABORT_TARGET_LENGTH + ) { + return value; + } + } + return null; +} + +/** + * Runs ahead of the chat abort route and answers for a background child's job, so the + * route's chat-shaped abort never reaches it. Every other request falls through untouched, + * including one for a job the caller does not own, which the route refuses itself. A job + * lookup that fails ends the request at the error boundary. + */ +export function createBackgroundChildAbortGuard(deps: { + getJob: (streamId: string) => Promise; +}): RequestHandler { + return async (req, res, next) => { + const streamId = abortTargetId(req.body); + const user = (req as Request & { user?: { id?: string; tenantId?: string } }).user; + if (streamId == null || user?.id == null) { + next(); + return; + } + let job: AbortGuardJob | null | undefined; + try { + job = await deps.getJob(streamId); + } catch (error) { + /** Fail closed: the route reads the job again, and a read that then succeeds would + * run its chat-shaped abort on a child this guard never got to see. */ + next(error); + return; + } + const metadata = job?.metadata; + const owned = + metadata?.userId === user.id && + (metadata.tenantId == null || metadata.tenantId === user.tenantId); + const refusal = owned ? backgroundChildJobRefusal(job) : null; + if (refusal == null) { + next(); + return; + } + res.status(refusal.status).json(refusal.body); + }; +} + +export function backgroundChildJobRefusal( + job: { metadata?: { subagentTaskId?: string } } | null | undefined, +): BackgroundChildJobRefusal | null { + const taskId = job?.metadata?.subagentTaskId; + if (taskId == null || taskId === '') return null; + return { + status: 409, + body: { code: CHILD_THREAD_READ_ONLY_CODE, error: CHILD_THREAD_READ_ONLY_ERROR }, + }; +} + interface SubagentTurnBody { conversationId?: unknown; arg?: { conversationId?: unknown }; diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index 935dce7bd72..64bb7f04b12 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -65,6 +65,7 @@ export * from './subagentThreads'; export * from './subagentActivity'; export * from './aggregation'; export * from './subagentContent'; +export * from './subagentGeneration'; export * from './subagentIdentity'; export * from './subagentCompletionWakeup'; export * from './subagentTaskRouting'; diff --git a/packages/api/src/agents/steering/__tests__/request.spec.ts b/packages/api/src/agents/steering/__tests__/request.spec.ts index 7884b4a09d1..533374a516e 100644 --- a/packages/api/src/agents/steering/__tests__/request.spec.ts +++ b/packages/api/src/agents/steering/__tests__/request.spec.ts @@ -183,6 +183,30 @@ describe('handleSteerRequest (real in-memory job manager)', () => { await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([]); }); + it('409s for a background child job and queues nothing', async () => { + const streamId = 'steer-req-background-child'; + await GenerationJobManager.createJob(streamId, user.id, streamId, { + initialMetadata: { subagentTaskId: 'task-1', subagentParentConversationId: 'parent' }, + }); + + const result = await handleSteerRequest(user, { conversationId: streamId, text: 'x' }); + + expect(result).toEqual({ status: 409, body: { code: 'CHILD_THREAD_READ_ONLY' } }); + await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([]); + }); + + it('403s for another user on a background child job before any other answer', async () => { + const streamId = 'steer-req-background-child-owner'; + await GenerationJobManager.createJob(streamId, 'someone-else', streamId, { + initialMetadata: { subagentTaskId: 'task-1', subagentParentConversationId: 'parent' }, + }); + + const result = await handleSteerRequest(user, { conversationId: streamId, text: 'x' }); + + expect(result.status).toBe(403); + expect(result.body.code).toBe('UNAUTHORIZED'); + }); + it('409s while the run is paused for human review', async () => { const streamId = 'steer-req-paused'; await GenerationJobManager.createJob(streamId, user.id); diff --git a/packages/api/src/agents/steering/request.ts b/packages/api/src/agents/steering/request.ts index 816a848124f..952e422a6ac 100644 --- a/packages/api/src/agents/steering/request.ts +++ b/packages/api/src/agents/steering/request.ts @@ -20,6 +20,7 @@ import { STEER_ENQUEUE_QUEUE_FULL, STEER_ENQUEUE_RECEIPT_FULL, } from '~/stream/interfaces/IJobStore'; +import { backgroundChildJobRefusal, CHILD_THREAD_READ_ONLY_CODE } from '~/agents/guard'; import { toSteerFileRef, collectFileIds, buildOwnerFilter } from './refs'; import { GenerationJobManager } from '~/stream/GenerationJobManager'; import { sanitizeFileForTransmit } from '../../utils/message'; @@ -544,6 +545,11 @@ async function handleSteerRequestInternal( if (expectedGeneration.value != null && job.createdAt !== expectedGeneration.value) { return { status: 409, body: { code: 'RUN_REPLACED' } }; } + /** The subagent task store runs a background child and injects guidance through its + * own control path; a steer queued here would be accepted and never drained. */ + if (backgroundChildJobRefusal(job) != null) { + return { status: 409, body: { code: CHILD_THREAD_READ_ONLY_CODE } }; + } /** A steer is model-bound instruction text for the ORIGINATING agent, so it * must clear the same role/ACL gates the chat path runs — revoked access diff --git a/packages/api/src/agents/subagentContent.spec.ts b/packages/api/src/agents/subagentContent.spec.ts index 04780b6f646..098e630969b 100644 --- a/packages/api/src/agents/subagentContent.spec.ts +++ b/packages/api/src/agents/subagentContent.spec.ts @@ -1,4 +1,4 @@ -import { ContentTypes, SteerEvents } from 'librechat-data-provider'; +import { ContentTypes, SteerEvents, StepEvents } from 'librechat-data-provider'; import type { SubagentContentAppendResult, SubagentContentMethods, @@ -6,7 +6,11 @@ import type { SubagentContentScope, } from '@librechat/data-schemas'; import type { SubagentUpdateEvent } from '@librechat/agents'; -import { readSubagentContent, SubagentContentJournal } from './subagentContent'; +import { + createSubagentContentFold, + SubagentContentJournal, + readSubagentContent, +} from './subagentContent'; const scope: SubagentContentScope = { user: 'owner', @@ -835,3 +839,106 @@ it('journals applied steering in its reserved slot before later normalized SDK s frame: { event: SteerEvents.ON_STEER_APPLIED, data: applied }, }); }); + +describe('createSubagentContentFold', () => { + type Frame = Parameters['apply']>[0]; + const frame = (event: string, data: unknown): Frame => ({ event, data }) as Frame; + const toolStep = frame(StepEvents.ON_RUN_STEP, { + id: 'tool', + index: 0, + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: 'call', name: 'lookup', args: '{}', type: 'tool_call' }], + }, + }); + + it('stamps a closed step onto its tool call', () => { + const fold = createSubagentContentFold(); + fold.apply(toolStep); + fold.apply( + frame(StepEvents.ON_RUN_STEP_CLOSED, { + id: 'tool', + index: 0, + type: 'tool_calls', + status: 'cancelled', + created_at: 1000, + closed_at: 1500, + }), + ); + + const part = fold.aggregator.contentParts[0] as { + type?: string; + tool_call?: { runStepStatus?: string; runStepClosedAt?: number }; + }; + expect(part.type).toBe(ContentTypes.TOOL_CALL); + expect(part.tool_call).toMatchObject({ runStepStatus: 'cancelled', runStepClosedAt: 1500 }); + }); + + it('ignores a close for a step it never saw', () => { + const fold = createSubagentContentFold(); + + expect(() => + fold.apply( + frame(StepEvents.ON_RUN_STEP_CLOSED, { + id: 'unknown', + index: 0, + type: 'tool_calls', + status: 'completed', + closed_at: 1, + }), + ), + ).not.toThrow(); + expect(fold.aggregator.contentParts).toHaveLength(0); + }); + + it('carries a commentary phase from the step onto its text', () => { + const fold = createSubagentContentFold(); + fold.apply( + frame(StepEvents.ON_RUN_STEP, { + id: 'note', + index: 0, + stepDetails: { + type: 'message_creation', + message_creation: { message_id: 'note', phase: 'commentary' }, + }, + }), + ); + fold.apply( + frame(StepEvents.ON_MESSAGE_DELTA, { + id: 'note', + delta: { content: [{ type: 'text', text: 'Checking the logs.' }] }, + }), + ); + + expect(fold.aggregator.contentParts[0]).toMatchObject({ + type: ContentTypes.TEXT, + text: 'Checking the logs.', + phase: 'commentary', + }); + }); + + it('places an applied steer in its reserved slot and keeps folding after it', () => { + const fold = createSubagentContentFold(); + const steer = { type: ContentTypes.STEER, steer: 'Focus on errors', steerId: 's1' }; + fold.apply(frame(SteerEvents.ON_STEER_APPLIED, { steerId: 's1', index: 0, part: steer })); + fold.apply( + frame(StepEvents.ON_RUN_STEP, { + id: 'after', + index: 1, + stepDetails: { type: 'message_creation', message_creation: { message_id: 'after' } }, + }), + ); + fold.apply( + frame(StepEvents.ON_MESSAGE_DELTA, { + id: 'after', + delta: { content: [{ type: 'text', text: 'On it.' }] }, + }), + ); + + expect(fold.aggregator.contentParts[0]).toEqual(steer); + expect(fold.aggregator.contentParts[1]).toMatchObject({ + type: ContentTypes.TEXT, + text: 'On it.', + }); + }); +}); diff --git a/packages/api/src/agents/subagentContent.ts b/packages/api/src/agents/subagentContent.ts index bef87338b86..b63cfdf6561 100644 --- a/packages/api/src/agents/subagentContent.ts +++ b/packages/api/src/agents/subagentContent.ts @@ -29,8 +29,8 @@ import type { import type { SubagentUpdateEvent } from '@librechat/agents'; import type { Response } from 'express'; import type { ServerRequest } from '~/types'; +import { createToolTimingAdapter, createToolTimingTracker } from './toolTiming'; import { toWireRunSteps } from '~/stream/interfaces/IJobStore'; -import { createToolTimingAdapter } from './toolTiming'; import { getSafeErrorMetadata } from '~/utils/errors'; const CONTENT_GRAPH_EVENTS = { @@ -380,6 +380,63 @@ export class SubagentContentJournal { } } +export type SubagentContentFold = { + readonly aggregator: ReturnType; + /** Synchronous, so a caller can fold a frame and publish it without yielding in between. */ + apply(frame: SubagentContentFrame): void; +}; + +/** Folds public child frames into content parts the way main chat aggregates its own stream. */ +export function createSubagentContentFold(): SubagentContentFold { + const aggregator = createContentAggregator(); + const tracker = createToolTimingTracker(); + const timing = createToolTimingAdapter({ emit: () => undefined }); + const apply = (frame: SubagentContentFrame): void => { + if (frame.event === SteerEvents.ON_STEER_APPLIED) { + aggregator.contentParts[frame.data.index] = frame.data.part; + return; + } + if (frame.event === StepEvents.ON_RUN_STEP_CLOSED) { + const step = aggregator.stepMap.get(frame.data.id); + if (step != null) step.status = frame.data.status; + const part = step == null ? undefined : aggregator.contentParts[step.index]; + if (part?.type === ContentTypes.TOOL_CALL && part.tool_call != null) { + part.tool_call.runStepStatus = frame.data.status; + Object.assign(part.tool_call, getRunStepCloseMetadata(frame.data)); + const duration = getRunStepDurationMs(frame.data); + if (duration != null) part.tool_call.runStepDurationMs = duration; + } + return; + } + if (frame.event === StepEvents.ON_RUN_STEP_DELTA) { + for (const marker of tracker.observe(frame.data)) { + timing.child(aggregator, { phase: 'tool_preparation', data: marker }); + } + } + /** SDK and wire enums have identical values, as in main-chat Redis recovery. */ + if ( + frame.event !== StepEvents.ON_TOOL_PREPARATION && + frame.event !== StepEvents.ON_TOOL_CALLS_DISPATCHED + ) { + aggregator.aggregateContent({ + event: CONTENT_GRAPH_EVENTS[frame.event], + data: frame.data, + } as Parameters[0]); + } + if (frame.event === StepEvents.ON_MESSAGE_DELTA) { + const step = aggregator.stepMap.get(frame.data.id); + const part = step == null ? undefined : aggregator.contentParts[step.index]; + if (part?.type === ContentTypes.TEXT && step?.stepDetails.type === 'message_creation') { + const creation = step.stepDetails.message_creation; + const phase = creation != null && 'phase' in creation ? creation.phase : undefined; + if (phase === 'commentary' || phase === 'final_answer') part.phase = phase; + } + } + timing.child(aggregator, { phase: frame.event.slice(3), data: frame.data }); + }; + return { aggregator, apply }; +} + type ContentReadMethods = Pick< SubagentContentMethods, 'getSubagentContentHead' | 'streamSubagentContentBatches' @@ -431,14 +488,8 @@ export async function readSubagentContent( }, }; } - const aggregator = createContentAggregator(); - const timing = createToolTimingAdapter({ - emit: (event) => - timing.child(aggregator, { - phase: event.event.slice(3), - data: event.data, - }), - }); + const fold = createSubagentContentFold(); + const { aggregator } = fold; const events: SubagentContentEvent[] = []; let throughSequence = afterSequence ?? -1; let revision: number | undefined; @@ -484,44 +535,7 @@ export async function readSubagentContent( events.push(entry); continue; } - const frame = entry.frame; - if (frame.event === SteerEvents.ON_STEER_APPLIED) { - aggregator.contentParts[frame.data.index] = frame.data.part; - continue; - } - if (frame.event === StepEvents.ON_RUN_STEP_CLOSED) { - const step = aggregator.stepMap.get(frame.data.id); - if (step != null) step.status = frame.data.status; - const part = step == null ? undefined : aggregator.contentParts[step.index]; - if (part?.type === ContentTypes.TOOL_CALL && part.tool_call != null) { - part.tool_call.runStepStatus = frame.data.status; - Object.assign(part.tool_call, getRunStepCloseMetadata(frame.data)); - const duration = getRunStepDurationMs(frame.data); - if (duration != null) part.tool_call.runStepDurationMs = duration; - } - continue; - } - if (frame.event === StepEvents.ON_RUN_STEP_DELTA) await timing.delta(frame.data); - /** SDK and wire enums have identical values, as in main-chat Redis recovery. */ - if ( - frame.event !== StepEvents.ON_TOOL_PREPARATION && - frame.event !== StepEvents.ON_TOOL_CALLS_DISPATCHED - ) { - aggregator.aggregateContent({ - event: CONTENT_GRAPH_EVENTS[frame.event], - data: frame.data, - } as Parameters[0]); - } - if (frame.event === StepEvents.ON_MESSAGE_DELTA) { - const step = aggregator.stepMap.get(frame.data.id); - const part = step == null ? undefined : aggregator.contentParts[step.index]; - if (part?.type === ContentTypes.TEXT && step?.stepDetails.type === 'message_creation') { - const creation = step.stepDetails.message_creation; - const phase = creation != null && 'phase' in creation ? creation.phase : undefined; - if (phase === 'commentary' || phase === 'final_answer') part.phase = phase; - } - } - timing.child(aggregator, { phase: frame.event.slice(3), data: frame.data }); + fold.apply(entry.frame); } throughSequence = batch.throughSequence; revision = batch.revision; diff --git a/packages/api/src/agents/subagentGeneration.spec.ts b/packages/api/src/agents/subagentGeneration.spec.ts new file mode 100644 index 00000000000..b8db0300def --- /dev/null +++ b/packages/api/src/agents/subagentGeneration.spec.ts @@ -0,0 +1,642 @@ +import { StepEvents } from 'librechat-data-provider'; +import type { SubagentContentFrame } from 'librechat-data-provider'; +import type { ChildGenerationInput, ChildGenerationManager } from './subagentGeneration'; +import type { ServerSentEvent } from '~/types'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { createChildGenerationHost, mergeDeltaFrames } from './subagentGeneration'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; + +jest.spyOn(console, 'log').mockImplementation(); + +const threadId = 'child-thread'; +const taskId = 'task-1'; + +const runStep = (id: string, index: number): SubagentContentFrame => + ({ + event: StepEvents.ON_RUN_STEP, + data: { + id, + index, + runId: 'sdk-run', + stepDetails: { type: 'message_creation', message_creation: { message_id: id } }, + }, + }) as SubagentContentFrame; + +const text = (id: string, value: string): SubagentContentFrame => + ({ + event: StepEvents.ON_MESSAGE_DELTA, + data: { id, delta: { content: [{ type: 'text', text: value }] } }, + }) as SubagentContentFrame; + +const reasoning = (id: string, value: string): SubagentContentFrame => + ({ + event: StepEvents.ON_REASONING_DELTA, + data: { id, delta: { content: [{ type: 'think', think: value }] } }, + }) as SubagentContentFrame; + +const final = { + requestMessage: { messageId: `${taskId}:user`, conversationId: threadId, text: 'Do the thing' }, + responseMessage: { messageId: `${taskId}:assistant`, conversationId: threadId, text: 'Done' }, +}; + +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +describe('child generation host', () => { + let manager: GenerationJobManagerClass; + + const input = (): ChildGenerationInput => ({ + userId: 'owner', + threadId, + taskId, + parentConversationId: 'parent-conversation', + agentId: 'agent_child', + sender: 'researcher', + userMessage: { messageId: `${taskId}:user`, parentMessageId: 'root', text: 'Do the thing' }, + }); + + beforeEach(() => { + manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }), + eventTransport: new InMemoryEventTransport(), + isRedis: false, + cleanupOnComplete: false, + }); + manager.initialize(); + }); + + afterEach(async () => { + await manager.destroy(); + }); + + it('opens a job keyed by the thread id and marked as a background child', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + + expect(generation).not.toBeNull(); + const job = await manager.getJob(threadId); + expect(job).toMatchObject({ status: 'running', createdAt: generation?.createdAt }); + expect(job?.metadata).toMatchObject({ + userId: 'owner', + conversationId: threadId, + responseMessageId: `${taskId}:assistant`, + subagentTaskId: taskId, + subagentParentConversationId: 'parent-conversation', + sender: 'researcher', + agent_id: 'agent_child', + }); + await expect(manager.getActiveJobIdsForUser('owner')).resolves.toEqual([]); + }); + + it('streams frames in order to a subscriber, with run steps under the response id', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + const received: ServerSentEvent[] = []; + await manager.subscribe(threadId, (event) => received.push(event)); + + generation?.emit(runStep('s1', 0)); + generation?.emit(text('s1', 'Hello')); + await flush(); + generation?.emit(text('s1', ' world')); + await flush(); + + const events = received.filter((event) => 'event' in event) as Array<{ + event: string; + data: Record; + }>; + expect(events.map((event) => event.event)).toEqual([ + StepEvents.ON_RUN_STEP, + StepEvents.ON_MESSAGE_DELTA, + StepEvents.ON_MESSAGE_DELTA, + ]); + expect(events[0].data.runId).toBe(`${taskId}:assistant`); + expect(generation?.content()).toEqual([expect.objectContaining({ type: 'text' })]); + expect(JSON.stringify(generation?.content())).toContain('Hello world'); + }); + + it('gives a late subscriber the content gathered so far', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + generation?.emit(runStep('s1', 0)); + generation?.emit(text('s1', 'Before you arrived')); + await flush(); + + const state = await manager.getResumeState(threadId, generation?.createdAt); + + expect(JSON.stringify(state?.aggregatedContent)).toContain('Before you arrived'); + expect(state?.userMessage).toMatchObject({ messageId: `${taskId}:user` }); + expect(state?.responseMessageId).toBe(`${taskId}:assistant`); + }); + + it('merges deltas that queue behind a slow write and loses none of the text', async () => { + let release: (() => void) | undefined; + const written: ServerSentEvent[] = []; + const slow: ChildGenerationManager = { + getJob: manager.getJob.bind(manager), + createJob: manager.createJob.bind(manager), + touchJob: manager.touchJob.bind(manager), + setContentParts: manager.setContentParts.bind(manager), + claimTerminalJob: manager.claimTerminalJob.bind(manager), + publishTerminalClaim: manager.publishTerminalClaim.bind(manager), + finishTerminalJob: manager.finishTerminalJob.bind(manager), + emitChunk: async (_streamId, event) => { + written.push(event); + await new Promise((resolve) => { + release = resolve; + }); + }, + }; + const generation = await createChildGenerationHost(slow).open(input()); + release?.(); + await flush(); + + generation?.emit(runStep('s1', 0)); + for (const word of ['a', 'b', 'c', 'd']) generation?.emit(text('s1', word)); + release?.(); + await flush(); + release?.(); + await flush(); + + const deltas = written.filter( + (event) => 'event' in event && event.event === StepEvents.ON_MESSAGE_DELTA, + ); + expect(deltas).toHaveLength(1); + expect(JSON.stringify(deltas[0])).toContain('abcd'); + expect(JSON.stringify(generation?.content())).toContain('abcd'); + }); + + it('publishes the final event only after the caller has persisted', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + const order: string[] = []; + let done: ServerSentEvent | undefined; + await manager.subscribe( + threadId, + () => undefined, + (event) => { + order.push('final'); + done = event; + }, + ); + + await generation?.settle(false, async () => { + const job = await manager.getJob(threadId); + order.push(`persist:${job?.status}:${job?.metadata.terminalPersistencePending}`); + return final; + }); + + expect(order).toEqual(['persist:complete:true', 'final']); + expect(done).toMatchObject({ + final: true, + conversation: { conversationId: threadId }, + requestMessage: { messageId: `${taskId}:user` }, + responseMessage: { messageId: `${taskId}:assistant`, text: 'Done' }, + }); + }); + + it('claims and publishes once when the caller settles again', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + const finals: ServerSentEvent[] = []; + await manager.subscribe( + threadId, + () => undefined, + (event) => finals.push(event), + ); + const claim = jest.spyOn(manager, 'claimTerminalJob'); + + await generation?.settle(false, async () => final); + const again = jest.fn(async () => null); + await generation?.settle(true, again); + + expect(claim).toHaveBeenCalledTimes(1); + expect(again).toHaveBeenCalledTimes(1); + expect(finals).toHaveLength(1); + expect(finals[0]).toMatchObject({ final: true, responseMessage: { text: 'Done' } }); + expect((await manager.getJob(threadId))?.status).toBe('complete'); + }); + + it('marks a cancelled child aborted in its final event', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + let done: ServerSentEvent | undefined; + await manager.subscribe( + threadId, + () => undefined, + (event) => { + done = event; + }, + ); + + await generation?.settle(true, async () => final); + + expect(done).toMatchObject({ final: true, aborted: true }); + }); + + it('tells a reader to refetch when persistence fails, and rethrows', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + let done: ServerSentEvent | undefined; + await manager.subscribe( + threadId, + () => undefined, + (event) => { + done = event; + }, + ); + + await expect( + generation?.settle(false, async () => { + throw new Error('mongo down: secret detail'); + }), + ).rejects.toThrow('mongo down'); + + expect(done).toMatchObject({ final: true, reconcile: true }); + expect(JSON.stringify(done)).not.toContain('secret detail'); + }); + + it('tells a reader to refetch when the caller had nothing to persist', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + let done: ServerSentEvent | undefined; + await manager.subscribe( + threadId, + () => undefined, + (event) => { + done = event; + }, + ); + + await expect(generation?.settle(true, async () => null)).resolves.toBeUndefined(); + + expect(done).toMatchObject({ final: true, reconcile: true }); + expect((await manager.getJob(threadId))?.status).toBe('aborted'); + }); + + it('keeps folding and still persists after its job is aborted from outside', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + generation?.emit(runStep('s1', 0)); + generation?.emit(text('s1', 'before ')); + await flush(); + + await manager.abortJob(threadId, { expectedCreatedAt: generation?.createdAt }); + expect(() => generation?.emit(text('s1', 'after'))).not.toThrow(); + await flush(); + + expect(JSON.stringify(generation?.content())).toContain('before after'); + const persist = jest.fn(async () => final); + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + expect(persist).toHaveBeenCalledTimes(1); + }); + + it.each([ + ['a chat', {}], + ['an event-bound child', { agentEventBindingId: 'binding-1' }], + ])('never replaces a live job that belongs to %s', async (_name, initialMetadata) => { + const live = await manager.createJob(threadId, 'owner', threadId, { initialMetadata }); + + await expect(createChildGenerationHost(manager).open(input())).resolves.toBeNull(); + + const job = await manager.getJob(threadId); + expect(job).toMatchObject({ status: 'running', createdAt: live.createdAt }); + expect(job?.metadata.subagentTaskId).toBeUndefined(); + }); + + it('replaces a background child job its owner left behind, and tells its readers', async () => { + const abandoned = await createChildGenerationHost(manager).open({ + ...input(), + taskId: 'crashed-task', + }); + let notice: ServerSentEvent | undefined; + await manager.subscribe( + threadId, + () => undefined, + (event) => { + notice = event; + }, + ); + + const generation = await createChildGenerationHost(manager).open(input()); + + expect(generation).not.toBeNull(); + expect(generation?.createdAt).toBeGreaterThan(abandoned?.createdAt ?? 0); + const job = await manager.getJob(threadId); + expect(job).toMatchObject({ status: 'running', createdAt: generation?.createdAt }); + expect(job?.metadata.subagentTaskId).toBe(taskId); + expect(notice).toMatchObject({ + final: true, + reconcile: true, + reconcileReason: 'generation_replaced', + }); + }); + + it('keeps folding content after a write fails, and still persists', async () => { + let fail = false; + const flaky: ChildGenerationManager = { + getJob: manager.getJob.bind(manager), + createJob: manager.createJob.bind(manager), + touchJob: manager.touchJob.bind(manager), + setContentParts: manager.setContentParts.bind(manager), + claimTerminalJob: manager.claimTerminalJob.bind(manager), + publishTerminalClaim: manager.publishTerminalClaim.bind(manager), + finishTerminalJob: manager.finishTerminalJob.bind(manager), + emitChunk: async (streamId, event, options) => { + if (fail) throw new Error('redis unavailable'); + await manager.emitChunk(streamId, event, options); + }, + }; + const generation = await createChildGenerationHost(flaky).open(input()); + generation?.emit(runStep('s1', 0)); + await flush(); + fail = true; + generation?.emit(text('s1', 'kept ')); + await flush(); + generation?.emit(text('s1', 'whole')); + + expect(JSON.stringify(generation?.content())).toContain('kept whole'); + let done: ServerSentEvent | undefined; + await manager.subscribe( + threadId, + () => undefined, + (event) => { + done = event; + }, + ); + const persist = jest.fn(async () => final); + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + expect(persist).toHaveBeenCalledTimes(1); + /** The stream stopped being written, but its job must not read as a live child. */ + expect((await manager.getJob(threadId))?.status).toBe('complete'); + expect(done).toMatchObject({ final: true, responseMessage: { text: 'Done' } }); + }); + + it('keeps a detached stream alive on touch until it is settled', async () => { + const flaky: ChildGenerationManager = { + getJob: manager.getJob.bind(manager), + createJob: manager.createJob.bind(manager), + touchJob: jest.fn(manager.touchJob.bind(manager)), + setContentParts: manager.setContentParts.bind(manager), + claimTerminalJob: manager.claimTerminalJob.bind(manager), + publishTerminalClaim: manager.publishTerminalClaim.bind(manager), + finishTerminalJob: manager.finishTerminalJob.bind(manager), + emitChunk: async () => { + throw new Error('redis unavailable'); + }, + }; + const generation = await createChildGenerationHost(flaky).open(input()); + await flush(); + + await generation?.touch(); + + expect(flaky.touchJob).toHaveBeenCalledWith(threadId, generation?.createdAt); + }); + + it('never throws into the child for a frame it cannot copy', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + const uncopyable = { + event: StepEvents.ON_RUN_STEP_COMPLETED, + data: { result: { id: 's1', callback: () => undefined, big: BigInt(1) } }, + } as unknown as SubagentContentFrame; + + generation?.emit(runStep('s1', 0)); + expect(() => generation?.emit(uncopyable)).not.toThrow(); + generation?.emit(text('s1', 'still folding')); + + expect(JSON.stringify(generation?.content())).toContain('still folding'); + const persist = jest.fn(async () => final); + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + expect(persist).toHaveBeenCalledTimes(1); + }); + + it('does not hold the result behind a terminal claim that never answers', async () => { + const stalled: ChildGenerationManager = { + getJob: manager.getJob.bind(manager), + createJob: manager.createJob.bind(manager), + touchJob: manager.touchJob.bind(manager), + setContentParts: manager.setContentParts.bind(manager), + emitChunk: manager.emitChunk.bind(manager), + publishTerminalClaim: manager.publishTerminalClaim.bind(manager), + finishTerminalJob: manager.finishTerminalJob.bind(manager), + claimTerminalJob: () => new Promise(() => undefined), + }; + const generation = await createChildGenerationHost(stalled, { settleTimeoutMs: 20 }).open( + input(), + ); + const persist = jest.fn(async () => final); + + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + + expect(persist).toHaveBeenCalledTimes(1); + }); + + it('gives up on a stream that takes too long to open, and ends the job if it arrives', async () => { + let createLate!: () => Promise; + const slow: ChildGenerationManager = { + getJob: manager.getJob.bind(manager), + touchJob: manager.touchJob.bind(manager), + setContentParts: manager.setContentParts.bind(manager), + emitChunk: manager.emitChunk.bind(manager), + claimTerminalJob: manager.claimTerminalJob.bind(manager), + publishTerminalClaim: manager.publishTerminalClaim.bind(manager), + finishTerminalJob: manager.finishTerminalJob.bind(manager), + createJob: (...args) => + new Promise((resolve, reject) => { + createLate = () => manager.createJob(...args).then(resolve, reject); + }), + }; + + await expect( + createChildGenerationHost(slow, { openTimeoutMs: 20 }).open(input()), + ).resolves.toBeNull(); + expect(await manager.getJob(threadId)).toBeUndefined(); + + await createLate(); + await flush(); + await flush(); + + expect((await manager.getJob(threadId))?.status).toBe('aborted'); + }); + + describe('when the job store misbehaves at settlement', () => { + const wrap = (overrides: Partial): ChildGenerationManager => ({ + getJob: manager.getJob.bind(manager), + createJob: manager.createJob.bind(manager), + touchJob: manager.touchJob.bind(manager), + setContentParts: manager.setContentParts.bind(manager), + emitChunk: manager.emitChunk.bind(manager), + claimTerminalJob: manager.claimTerminalJob.bind(manager), + publishTerminalClaim: manager.publishTerminalClaim.bind(manager), + finishTerminalJob: manager.finishTerminalJob.bind(manager), + ...overrides, + }); + + it('releases a terminal claim that lands after the wait, with a refetch notice', async () => { + let land!: () => void; + const landed = new Promise((resolve) => { + land = resolve; + }); + const late = wrap({ + claimTerminalJob: async (...args) => { + await landed; + return manager.claimTerminalJob(...args); + }, + }); + const generation = await createChildGenerationHost(late, { settleTimeoutMs: 20 }).open( + input(), + ); + let done: ServerSentEvent | undefined; + await manager.subscribe( + threadId, + () => undefined, + (event) => { + done = event; + }, + ); + const persist = jest.fn(async () => final); + + await generation?.settle(false, persist); + expect(persist).toHaveBeenCalledTimes(1); + expect(done).toBeUndefined(); + expect((await manager.getJob(threadId))?.status).toBe('running'); + + land(); + await flush(); + await flush(); + + expect(done).toMatchObject({ final: true, reconcile: true }); + expect((await manager.getJob(threadId))?.status).toBe('complete'); + }); + + it('persists and returns when the terminal claim throws', async () => { + const failing = wrap({ + claimTerminalJob: async () => { + throw new Error('redis unavailable'); + }, + }); + const generation = await createChildGenerationHost(failing).open(input()); + const persist = jest.fn(async () => final); + + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + + expect(persist).toHaveBeenCalledTimes(1); + }); + + it('still releases the job when publishing the final event throws', async () => { + const finish = jest.fn(manager.finishTerminalJob.bind(manager)); + const failing = wrap({ + publishTerminalClaim: async () => { + throw new Error('redis unavailable'); + }, + finishTerminalJob: finish, + }); + const generation = await createChildGenerationHost(failing).open(input()); + const persist = jest.fn(async () => final); + + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + + expect(persist).toHaveBeenCalledTimes(1); + expect(finish).toHaveBeenCalledTimes(1); + }); + + it('gives up waiting on a write that never lands, and keeps the content whole', async () => { + let writes = 0; + const stalled = wrap({ + emitChunk: () => { + writes += 1; + return new Promise(() => undefined); + }, + }); + const generation = await createChildGenerationHost(stalled, { settleTimeoutMs: 20 }).open( + input(), + ); + generation?.emit(runStep('s1', 0)); + generation?.emit(text('s1', 'never ')); + generation?.emit(text('s1', 'written')); + const persist = jest.fn(async () => final); + + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + + expect(writes).toBe(1); + expect(persist).toHaveBeenCalledTimes(1); + expect(JSON.stringify(generation?.content())).toContain('never written'); + }); + + it('stops queueing past its backlog limit without losing content', async () => { + let writes = 0; + const stalled = wrap({ + emitChunk: () => { + writes += 1; + return new Promise(() => undefined); + }, + }); + const generation = await createChildGenerationHost(stalled, { settleTimeoutMs: 20 }).open( + input(), + ); + /** Distinct steps never merge, so each one is its own queued frame. */ + for (let index = 0; index < 5_002; index += 1) { + generation?.emit(runStep(`step-${index}`, index)); + } + generation?.emit(text('step-5001', 'after the limit')); + + expect(writes).toBe(1); + expect(JSON.stringify(generation?.content())).toContain('after the limit'); + const persist = jest.fn(async () => final); + await expect(generation?.settle(false, persist)).resolves.toBeUndefined(); + expect(persist).toHaveBeenCalledTimes(1); + }); + + it('does not throw from touch when the store rejects it', async () => { + const failing = wrap({ + touchJob: async () => { + throw new Error('redis unavailable'); + }, + }); + const generation = await createChildGenerationHost(failing).open(input()); + + await expect(generation?.touch()).resolves.toBeUndefined(); + }); + }); + + it('refreshes the idle clock on touch', async () => { + const generation = await createChildGenerationHost(manager).open(input()); + const touch = jest.spyOn(manager, 'touchJob'); + + await generation?.touch(); + + expect(touch).toHaveBeenCalledWith(threadId, generation?.createdAt); + await expect(touch.mock.results[0].value).resolves.toBe(true); + }); +}); + +describe('mergeDeltaFrames', () => { + it('joins two text deltas for one step', () => { + expect(mergeDeltaFrames(text('s1', 'ab'), text('s1', 'cd'))).toEqual(text('s1', 'abcd')); + }); + + it('joins two reasoning deltas for one step', () => { + expect(mergeDeltaFrames(reasoning('s1', 'ab'), reasoning('s1', 'cd'))).toEqual( + reasoning('s1', 'abcd'), + ); + }); + + it('leaves deltas for different steps or kinds apart', () => { + expect(mergeDeltaFrames(text('s1', 'a'), text('s2', 'b'))).toBeNull(); + expect(mergeDeltaFrames(text('s1', 'a'), reasoning('s1', 'b'))).toBeNull(); + expect(mergeDeltaFrames(runStep('s1', 0), text('s1', 'b'))).toBeNull(); + }); + + it('leaves a delta with any extra field apart', () => { + const tagged = { + event: StepEvents.ON_MESSAGE_DELTA, + data: { id: 's1', delta: { content: [{ type: 'text', text: 'a', index: 2 }] } }, + } as unknown as SubagentContentFrame; + const multi = { + event: StepEvents.ON_MESSAGE_DELTA, + data: { + id: 's1', + delta: { + content: [ + { type: 'text', text: 'a' }, + { type: 'text', text: 'b' }, + ], + }, + }, + } as unknown as SubagentContentFrame; + + expect(mergeDeltaFrames(tagged, text('s1', 'b'))).toBeNull(); + expect(mergeDeltaFrames(text('s1', 'a'), multi)).toBeNull(); + }); +}); diff --git a/packages/api/src/agents/subagentGeneration.ts b/packages/api/src/agents/subagentGeneration.ts new file mode 100644 index 00000000000..5f5a4d5c982 --- /dev/null +++ b/packages/api/src/agents/subagentGeneration.ts @@ -0,0 +1,471 @@ +import { logger } from '@librechat/data-schemas'; +import { EModelEndpoint, StepEvents } from 'librechat-data-provider'; +import type { Agents, SubagentContentFrame, TMessageContentParts } from 'librechat-data-provider'; +import type { GenerationJobManagerClass, TerminalJobClaim } from '~/stream/GenerationJobManager'; +import type { FinalMessageFields, ServerSentEvent } from '~/types'; +import { createSubagentContentFold } from './subagentContent'; + +/** Frames waiting on a slow store before the stream is given up, matching the manager's own + * early-event ceiling. Merged deltas do not count twice, so only distinct steps reach it. */ +const MAX_QUEUED_FRAMES = 5_000; +/** How long the child waits on its stream's store before going on without it. Both bound a + * best-effort live view whose fallback is the child's own persisted result, so they are + * internal hygiene and not operator levers. */ +const DEFAULT_SETTLE_TIMEOUT_MS = 1_000; +const DEFAULT_OPEN_TIMEOUT_MS = 3_000; + +const EXPIRED = Symbol('expired'); + +/** Resolves with the operation, or with `EXPIRED` once it has taken too long. The operation + * keeps running; a rejection after expiry is swallowed. */ +async function within(operation: Promise, timeoutMs: number): Promise { + let timer: ReturnType | undefined; + const expired = new Promise((resolve) => { + timer = setTimeout(() => resolve(EXPIRED), timeoutMs); + timer.unref?.(); + }); + try { + return await Promise.race([operation, expired]); + } finally { + if (timer != null) clearTimeout(timer); + operation.catch(() => undefined); + } +} + +/** The producer reuses its event objects, so the stream keeps its own copy. */ +function ownCopy(frame: SubagentContentFrame): SubagentContentFrame | null { + try { + return structuredClone(frame); + } catch { + try { + return JSON.parse(JSON.stringify(frame)) as SubagentContentFrame; + } catch { + return null; + } + } +} + +/** The part of the generation job manager a child's stream needs. */ +export type ChildGenerationManager = Pick< + GenerationJobManagerClass, + | 'getJob' + | 'createJob' + | 'emitChunk' + | 'touchJob' + | 'setContentParts' + | 'claimTerminalJob' + | 'publishTerminalClaim' + | 'finishTerminalJob' +>; + +export type ChildGenerationInput = { + userId: string; + /** The child thread's conversation id, which is also the job's stream id. */ + threadId: string; + taskId: string; + parentConversationId: string; + agentId?: string; + sender: string; + userMessage: { messageId: string; parentMessageId: string; text: string }; +}; + +export type ChildGenerationFinal = { + requestMessage: FinalMessageFields; + responseMessage: FinalMessageFields; +}; + +/** + * One background child's resumable stream. It observes the run and never controls it: a stream + * that cannot be opened or written, or a job that is aborted, reaped or retired after a failed + * write, degrades the live view and leaves the run and its result alone. The task store stops a + * child through its own cancellation, which deletion, shutdown and the control endpoint all use. + */ +export interface ChildGeneration { + readonly createdAt: number; + /** Never blocks the caller. While a write is in flight, text deltas for one step merge. */ + emit(frame: SubagentContentFrame): void; + /** Refreshes the job's idle clock while the child works without output. */ + touch(): Promise; + /** Every frame emitted so far, folded into content parts. */ + content(): TMessageContentParts[]; + /** + * Ends the stream around the caller's own persistence: the terminal state is claimed + * first, `persist` writes the child's rows, and the final event is published only after + * they exist. `persist` always runs and its failure is rethrown. When it writes nothing + * and returns null, or throws, readers are told to refetch instead of being handed a final. + */ + settle(aborted: boolean, persist: () => Promise): Promise; +} + +export interface ChildGenerationHost { + /** + * Opens the stream for a run whose caller holds the child thread's lease. Null when the + * stream could not be opened; the child runs without one. + */ + open(input: ChildGenerationInput): Promise; +} + +type QueuedEvent = { event: ServerSentEvent; frame?: SubagentContentFrame }; + +const TEXT_KEYS: Readonly> = { text: 'text', think: 'think' }; + +type DeltaFrame = Extract< + SubagentContentFrame, + { event: StepEvents.ON_MESSAGE_DELTA | StepEvents.ON_REASONING_DELTA } +>; + +const isDeltaFrame = (frame: SubagentContentFrame): frame is DeltaFrame => + frame.event === StepEvents.ON_MESSAGE_DELTA || frame.event === StepEvents.ON_REASONING_DELTA; + +/** The one string a plain single-part delta carries, or null for any other shape. */ +function deltaText(frame: DeltaFrame): { type: string; key: string; value: string } | null { + const data = frame.data as { id?: unknown; delta?: { content?: unknown } }; + if (Object.keys(data).length !== 2 || data.delta == null) return null; + if (Object.keys(data.delta).length !== 1 || !Array.isArray(data.delta.content)) return null; + if (data.delta.content.length !== 1) return null; + const part = data.delta.content[0] as Record | null; + if (part == null || typeof part.type !== 'string' || Object.keys(part).length !== 2) return null; + const key = TEXT_KEYS[part.type]; + const value = key == null ? undefined : part[key]; + return typeof value === 'string' ? { type: part.type, key, value } : null; +} + +/** + * Two consecutive deltas for one step become a single frame carrying both strings, which + * every reader folds to the same content. Anything but a plain text or reasoning delta + * is left alone. + */ +export function mergeDeltaFrames( + previous: SubagentContentFrame, + next: SubagentContentFrame, +): SubagentContentFrame | null { + if (!isDeltaFrame(previous) || !isDeltaFrame(next) || previous.event !== next.event) return null; + if (previous.data.id !== next.data.id) return null; + const left = deltaText(previous); + const right = deltaText(next); + if (left == null || right == null || left.type !== right.type) return null; + return { + event: previous.event, + data: { + id: previous.data.id, + delta: { content: [{ type: left.type, [left.key]: left.value + right.value }] }, + }, + } as SubagentContentFrame; +} + +class ManagedChildGeneration implements ChildGeneration { + private readonly fold = createSubagentContentFold(); + private readonly queue: QueuedEvent[] = []; + private draining?: Promise; + /** The stream is no longer written; frames are still folded so `content` stays whole. */ + private detached = false; + private settling = false; + /** This stream claimed its job's terminal state; a later settlement only persists. */ + private ended = false; + + constructor( + private readonly manager: ChildGenerationManager, + private readonly input: ChildGenerationInput, + readonly createdAt: number, + private readonly responseMessageId: string, + private readonly settleTimeoutMs: number, + ) {} + + enqueue(event: ServerSentEvent, frame?: SubagentContentFrame): void { + this.queue.push({ event, ...(frame == null ? {} : { frame }) }); + this.pump(); + } + + emit(frame: SubagentContentFrame): void { + if (this.settling) return; + const own = ownCopy(frame); + if (own == null) { + logger.warn('[subagentGeneration] Child frame could not be copied; detaching', { + threadId: this.input.threadId, + taskId: this.input.taskId, + event: frame.event, + }); + this.detach(); + /** Folding reads the frame now, so the content stays whole without a copy to send. */ + this.apply(frame); + return; + } + if (this.detached) { + this.apply(own); + return; + } + const last = this.queue[this.queue.length - 1]; + const merged = last?.frame == null ? null : mergeDeltaFrames(last.frame, own); + if (merged != null) { + this.queue[this.queue.length - 1] = { event: this.toEvent(merged), frame: merged }; + return; + } + if (this.queue.length >= MAX_QUEUED_FRAMES) { + logger.warn('[subagentGeneration] Child stream backlog exceeded its limit; detaching', { + threadId: this.input.threadId, + taskId: this.input.taskId, + }); + this.detach(); + this.apply(own); + return; + } + this.enqueue(this.toEvent(own), own); + } + + /** A detached stream is touched too: its job is still the record that the child is running, + * and it is ended at settlement like any other. */ + async touch(): Promise { + if (this.settling) return; + await this.manager.touchJob(this.input.threadId, this.createdAt).catch(() => false); + } + + content(): TMessageContentParts[] { + return this.fold.aggregator.contentParts as TMessageContentParts[]; + } + + /** The live array the manager snapshots for a resume on this replica. */ + parts(): Agents.MessageContentComplex[] { + return this.fold.aggregator.contentParts as Agents.MessageContentComplex[]; + } + + async settle( + aborted: boolean, + persist: () => Promise, + ): Promise { + this.settling = true; + if (this.ended) { + await persist(); + return; + } + await this.drainWithin(this.settleTimeoutMs); + /** A detached stream is claimed too. Its final event carries the whole response, and a + * job left running would read as a live child long after this one ended. */ + const claim = await this.claimWithin(aborted); + this.ended = claim != null; + let final: ChildGenerationFinal | null; + try { + final = await persist(); + } catch (error) { + if (claim != null) await within(this.publish(claim, null), this.settleTimeoutMs); + throw error; + } + if (claim == null) return; + await within( + this.publish( + claim, + final == null + ? null + : { + final: true, + conversation: { conversationId: this.input.threadId }, + requestMessage: final.requestMessage, + responseMessage: final.responseMessage, + ...(aborted ? { aborted: true } : {}), + }, + ), + this.settleTimeoutMs, + ); + } + + /** The child's result is never held behind a stalled store. A claim that lands after the + * wait is released with a refetch notice, since nothing will publish its final event. */ + private async claimWithin(aborted: boolean): Promise { + const claiming = this.claim(aborted); + const claim = await within(claiming, this.settleTimeoutMs); + if (claim !== EXPIRED) return claim; + logger.warn('[subagentGeneration] Child stream terminal claim timed out', { + threadId: this.input.threadId, + taskId: this.input.taskId, + }); + void claiming.then((late) => (late == null ? undefined : this.publish(late, null))); + return null; + } + + private toEvent(frame: SubagentContentFrame): ServerSentEvent { + const data = + frame.event === StepEvents.ON_RUN_STEP + ? { ...frame.data, runId: this.responseMessageId } + : frame.data; + return { event: frame.event, data: data as unknown as Record }; + } + + private apply(frame: SubagentContentFrame): void { + try { + this.fold.apply(frame); + } catch (error) { + logger.warn('[subagentGeneration] Failed to fold a child frame', error); + } + } + + /** Stops writing and folds whatever was still waiting, in order. */ + private detach(): void { + this.detached = true; + for (const entry of this.queue.splice(0)) { + if (entry.frame != null) this.apply(entry.frame); + } + } + + private pump(): void { + if (this.draining != null) return; + const draining = this.drain().finally(() => { + if (this.draining === draining) this.draining = undefined; + if (this.queue.length > 0 && !this.detached) this.pump(); + }); + this.draining = draining; + } + + private async drain(): Promise { + while (this.queue.length > 0 && !this.detached) { + const entry = this.queue.shift()!; + /** Fold and publish in one synchronous step: a resume snapshot taken between them + * would carry this frame's content and then receive the frame again. */ + if (entry.frame != null) this.apply(entry.frame); + try { + await this.manager.emitChunk(this.input.threadId, entry.event, { + expectedCreatedAt: this.createdAt, + }); + } catch (error) { + logger.warn('[subagentGeneration] Child stream write failed; detaching', error); + this.detach(); + } + } + } + + private async drainWithin(timeoutMs: number): Promise { + let timer: ReturnType | undefined; + const expired = new Promise<'expired'>((resolve) => { + timer = setTimeout(() => resolve('expired'), timeoutMs); + timer.unref?.(); + }); + try { + while (this.draining != null && !this.detached) { + if ((await Promise.race([this.draining, expired])) === 'expired') { + logger.warn('[subagentGeneration] Child stream did not drain before settlement', { + threadId: this.input.threadId, + taskId: this.input.taskId, + }); + this.detach(); + } + } + } finally { + if (timer != null) clearTimeout(timer); + } + } + + private async claim(aborted: boolean): Promise { + try { + return await this.manager.claimTerminalJob( + this.input.threadId, + aborted ? 'aborted' : 'complete', + undefined, + this.createdAt, + { persistencePending: true }, + ); + } catch (error) { + logger.warn('[subagentGeneration] Failed to claim the child stream terminal state', error); + return null; + } + } + + private async publish(claim: TerminalJobClaim, event: ServerSentEvent | null): Promise { + try { + await this.manager.publishTerminalClaim(claim, event); + } catch (error) { + logger.warn('[subagentGeneration] Failed to publish the child stream final event', error); + } finally { + await this.manager.finishTerminalJob(claim).catch((error) => { + logger.warn('[subagentGeneration] Failed to release the child stream job', error); + }); + } + } +} + +/** Runs each background child as a generation job keyed by its thread id. */ +export function createChildGenerationHost( + manager: ChildGenerationManager, + options: { settleTimeoutMs?: number; openTimeoutMs?: number } = {}, +): ChildGenerationHost { + /** Ends a job created after its child stopped waiting for it, so it does not read as live. */ + const discard = async (streamId: string, createdAt: number): Promise => { + try { + const claim = await manager.claimTerminalJob(streamId, 'aborted', undefined, createdAt, { + persistencePending: true, + }); + if (claim == null) return; + try { + await manager.publishTerminalClaim(claim, null); + } finally { + await manager.finishTerminalJob(claim); + } + } catch (error) { + logger.warn('[subagentGeneration] Failed to discard a late child stream', error); + } + }; + return { + async open(input) { + const responseMessageId = `${input.taskId}:assistant`; + const userMessage = { + messageId: input.userMessage.messageId, + parentMessageId: input.userMessage.parentMessageId, + conversationId: input.threadId, + text: input.userMessage.text, + }; + try { + /** + * The caller holds the thread's lease, so a background child's job still marked live + * here has lost its owner: a crashed replica, or a settlement that could not reach the + * store. It is replaced, and its readers are told to refetch. Any other live job is + * someone else's and is never replaced: an event-bound child paused for approval keeps + * its job without holding the lease. + */ + const abandoned = await within( + manager.getJob(input.threadId).then((job) => job?.metadata.subagentTaskId != null), + options.openTimeoutMs ?? DEFAULT_OPEN_TIMEOUT_MS, + ); + if (abandoned === EXPIRED) throw new Error('child_stream_open_timeout'); + const creating = manager.createJob(input.threadId, input.userId, input.threadId, { + ...(abandoned ? {} : { rejectActivePredecessor: true }), + initialMetadata: { + conversationId: input.threadId, + endpoint: EModelEndpoint.agents, + sender: input.sender, + responseMessageId, + userMessage, + subagentTaskId: input.taskId, + subagentParentConversationId: input.parentConversationId, + ...(input.agentId == null ? {} : { agent_id: input.agentId }), + }, + }); + const job = await within(creating, options.openTimeoutMs ?? DEFAULT_OPEN_TIMEOUT_MS); + if (job === EXPIRED) { + void creating.then( + (late) => discard(input.threadId, late.createdAt), + () => undefined, + ); + throw new Error('child_stream_open_timeout'); + } + const generation = new ManagedChildGeneration( + manager, + input, + job.createdAt, + responseMessageId, + options.settleTimeoutMs ?? DEFAULT_SETTLE_TIMEOUT_MS, + ); + manager.setContentParts(input.threadId, generation.parts(), job.createdAt); + generation.enqueue({ + created: true, + message: { ...userMessage, sender: 'User', isCreatedByUser: true }, + streamId: input.threadId, + }); + return generation; + } catch (error) { + logger.warn('[subagentGeneration] Failed to open a child stream; running without one', { + threadId: input.threadId, + taskId: input.taskId, + error: error instanceof Error ? error.name : typeof error, + }); + return null; + } + }, + }; +} diff --git a/packages/api/src/agents/subagentThreads.generation.spec.ts b/packages/api/src/agents/subagentThreads.generation.spec.ts new file mode 100644 index 00000000000..7379e3f4402 --- /dev/null +++ b/packages/api/src/agents/subagentThreads.generation.spec.ts @@ -0,0 +1,518 @@ +import mongoose from 'mongoose'; +import { randomUUID } from 'node:crypto'; +import { EModelEndpoint } from 'librechat-data-provider'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { createModels, createMethods, logger } from '@librechat/data-schemas'; +import { AIMessage, HumanMessage } from '@librechat/agents/langchain/messages'; +import type { + SubagentTaskStartRequest, + SubagentTaskStartResult, + SubagentTaskRuntime, + SubagentUpdateEvent, +} from '@librechat/agents'; +import type { AllMethods, IConversation, IMessage } from '@librechat/data-schemas'; +import type { FinalEvent, ServerSentEvent } from '~/types'; +import { buildSubagentThreadTaskConfig, SubagentThreadTaskStore } from './subagentThreads'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { createChildGenerationHost } from './subagentGeneration'; + +jest.spyOn(console, 'log').mockImplementation(); + +let mongod: MongoMemoryServer; +let methods: AllMethods; +let manager: GenerationJobManagerClass; +let loggerErrorSpy: jest.SpyInstance; + +const userId = 'child-stream-user'; + +const progress = ( + phase: SubagentUpdateEvent['phase'], + data: SubagentUpdateEvent['data'], +): SubagentUpdateEvent => ({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher-agent', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [], + phase, + data, + timestamp: '2026-11-14T20:00:00.000Z', +}); + +const step = () => + progress('run_step', { + id: 'step-1', + index: 0, + stepDetails: { type: 'message_creation', message_creation: { message_id: 'step-1' } }, + }); + +const words = (text: string) => + progress('message_delta', { id: 'step-1', delta: { content: [{ type: 'text', text }] } }); + +/** A child that streams, then waits for the test before it finishes or fails. */ +function gatedRun(outcome: 'complete' | 'fail' = 'complete') { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const run: SubagentTaskStartRequest['run'] = async ( + runtime: SubagentTaskRuntime, + initialMessages = [], + ) => { + runtime.reportProgress(step()); + runtime.reportProgress(words('Looking ')); + runtime.reportProgress(words('into it.')); + await Promise.race([ + gate, + new Promise((_resolve, reject) => { + runtime.signal.addEventListener('abort', () => reject(runtime.signal.reason), { + once: true, + }); + }), + ]); + if (outcome === 'fail') throw new Error('provider exploded with key sk-secret'); + return { + content: 'Looking into it.', + messages: [ + ...initialMessages, + new HumanMessage('Investigate the issue.'), + new AIMessage('Looking into it.'), + ], + }; + }; + return { run, release }; +} + +function taskRequest( + scopeId: string, + overrides: Partial, +): SubagentTaskStartRequest { + return { + scopeId, + idempotencyKey: randomUUID(), + parentRunId: randomUUID(), + parentAgentId: 'parent-agent', + parentToolCallId: randomUUID(), + input: 'Investigate the issue.', + subagentKind: 'agent', + subagentType: 'researcher-agent', + run: gatedRun().run, + ...overrides, + }; +} + +function threadOf(started: SubagentTaskStartResult): { threadId: string; taskId: string } { + if (!started.accepted || !started.task.threadId) throw new Error('Expected an accepted task.'); + return { threadId: started.task.threadId, taskId: started.task.taskId }; +} + +async function waitUntil(condition: () => boolean | Promise, what: string): Promise { + for (let attempt = 0; attempt < 400; attempt += 1) { + if (await condition()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Timed out waiting for ${what}.`); +} + +async function setup(activity?: { generationJobs?: boolean; generationContentBytes?: number }) { + const parentConversationId = randomUUID(); + await methods.saveConvo( + { userId }, + { + conversationId: parentConversationId, + endpoint: EModelEndpoint.agents, + title: 'Parent thread', + agent_id: 'parent-agent', + }, + ); + const store = new SubagentThreadTaskStore(methods, { + childGenerations: createChildGenerationHost(manager), + ...(activity == null ? {} : { activity }), + }); + const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + return { store, scopeId: config.scopeId, parentConversationId }; +} + +/** Attaches the way the stream route does, collecting chunks and the final event. */ +async function attach(threadId: string) { + const chunks: ServerSentEvent[] = []; + let resolveFinal!: (event: FinalEvent) => void; + const final = new Promise((resolve) => { + resolveFinal = resolve; + }); + await manager.subscribe( + threadId, + (event) => chunks.push(event), + (event) => resolveFinal(event as FinalEvent), + ); + return { chunks, final }; +} + +const assistantRow = async (threadId: string, taskId: string) => + ( + await methods.getMessages( + { user: userId, conversationId: threadId, messageId: `${taskId}:assistant` }, + '+subagentTask', + ) + )[0]; + +beforeAll(async () => { + loggerErrorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger); + mongod = await MongoMemoryServer.create(); + createModels(mongoose); + methods = createMethods(mongoose); + await mongoose.connect(mongod.getUri()); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongod?.stop(); + loggerErrorSpy.mockRestore(); +}); + +beforeEach(async () => { + await Promise.all([ + (mongoose.models.Message as mongoose.Model).deleteMany({}), + (mongoose.models.Conversation as mongoose.Model).deleteMany({}), + ]); + manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }), + eventTransport: new InMemoryEventTransport(), + isRedis: false, + cleanupOnComplete: false, + }); + manager.initialize(); +}); + +afterEach(async () => { + await manager.destroy(); +}); + +describe('a background child as a generation job', () => { + it('runs under a job keyed by its thread id that is not listed as a chat', async () => { + const { store, scopeId, parentConversationId } = await setup(); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + + const job = await manager.getJob(threadId); + expect(job).toMatchObject({ status: 'running' }); + expect(job?.metadata).toMatchObject({ + userId, + conversationId: threadId, + subagentTaskId: taskId, + subagentParentConversationId: parentConversationId, + responseMessageId: `${taskId}:assistant`, + }); + await expect(manager.getActiveJobIdsForUser(userId)).resolves.toEqual([]); + + release(); + await waitUntil(async () => (await manager.getJob(threadId))?.status === 'complete', 'settle'); + }); + + it('hands a reader who opens mid-run everything streamed so far', async () => { + const { store, scopeId } = await setup(); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil( + async () => + JSON.stringify((await manager.getResumeState(threadId))?.aggregatedContent ?? []).includes( + 'into it.', + ), + 'streamed content', + ); + + const state = await manager.getResumeState(threadId); + + expect(JSON.stringify(state?.aggregatedContent)).toContain('Looking into it.'); + expect(state?.userMessage).toMatchObject({ + messageId: `${taskId}:user`, + text: 'Investigate the issue.', + }); + expect(state?.responseMessageId).toBe(`${taskId}:assistant`); + expect(state?.runSteps).toHaveLength(1); + + release(); + await waitUntil(async () => (await manager.getJob(threadId))?.status === 'complete', 'settle'); + }); + + it('publishes the final event only once the result row is durable', async () => { + const { store, scopeId } = await setup(); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + const { final } = await attach(threadId); + const rowAtFinal = final.then(() => assistantRow(threadId, taskId)); + + release(); + + await expect(final).resolves.toMatchObject({ + final: true, + conversation: { conversationId: threadId }, + requestMessage: { messageId: `${taskId}:user`, text: 'Investigate the issue.' }, + responseMessage: { + messageId: `${taskId}:assistant`, + parentMessageId: `${taskId}:user`, + text: 'Looking into it.', + unfinished: false, + }, + }); + expect(JSON.stringify((await final).responseMessage?.content)).toContain('Looking into it.'); + expect((await rowAtFinal)?.subagentTask?.status).toBe('completed'); + }); + + it('ends a cancelled child with an aborted final and a cancelled row', async () => { + const { store, scopeId } = await setup(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, {}))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + const { final } = await attach(threadId); + + store.control(scopeId, taskId, { action: 'cancel' }); + + await expect(final).resolves.toMatchObject({ final: true, aborted: true }); + expect((await assistantRow(threadId, taskId))?.subagentTask?.status).toBe('cancelled'); + expect((await manager.getJob(threadId))?.status).toBe('aborted'); + }); + + it('ends a failed child with an error row and no provider detail', async () => { + const { store, scopeId } = await setup(); + const { run, release } = gatedRun('fail'); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + const { final } = await attach(threadId); + + release(); + + const event = await final; + expect(event).toMatchObject({ final: true, responseMessage: { error: true } }); + expect(JSON.stringify(event)).not.toContain('sk-secret'); + expect((await assistantRow(threadId, taskId))?.error).toBe(true); + }); + + it('lets the child finish and persist after its job is aborted from outside', async () => { + const { store, scopeId } = await setup(); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + const job = await manager.getJob(threadId); + + await manager.abortJob(threadId, { expectedCreatedAt: job?.createdAt }); + /** The stream is over; the run is the task store's to stop, and nobody stopped it. */ + expect(store.get(scopeId, taskId)?.status).toBe('running'); + release(); + + await waitUntil(() => store.get(scopeId, taskId)?.status === 'completed', 'the task'); + const row = await assistantRow(threadId, taskId); + expect(row?.subagentTask?.status).toBe('completed'); + expect(row?.text).toBe('Looking into it.'); + }); + + it('still persists a cancellation after the job manager has shut down', async () => { + const { store, scopeId } = await setup(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, {}))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + + await manager.destroy(); + expect(store.get(scopeId, taskId)?.status).toBe('running'); + store.control(scopeId, taskId, { action: 'cancel' }); + + await waitUntil( + async () => (await assistantRow(threadId, taskId))?.subagentTask?.status === 'cancelled', + 'the cancelled row', + ); + expect(store.get(scopeId, taskId)?.status).toBe('cancelled'); + expect((await assistantRow(threadId, taskId))?.text).toBe('Subagent task was cancelled.'); + }); + + it('gives a continuation of the thread a new generation of the same stream', async () => { + const { store, scopeId } = await setup(); + const first = gatedRun(); + const started = threadOf(store.start(taskRequest(scopeId, { run: first.run }))); + await waitUntil(async () => (await manager.getJob(started.threadId)) != null, 'the first job'); + const firstCreatedAt = (await manager.getJob(started.threadId))?.createdAt ?? 0; + first.release(); + await waitUntil(() => store.get(scopeId, started.taskId)?.status === 'completed', 'first task'); + + const second = gatedRun(); + const continued = threadOf( + store.start(taskRequest(scopeId, { run: second.run, threadId: started.threadId })), + ); + await waitUntil( + async () => + (await manager.getJob(started.threadId))?.metadata.subagentTaskId === continued.taskId, + 'the continuation job', + ); + + const job = await manager.getJob(started.threadId); + expect(continued.threadId).toBe(started.threadId); + expect(job).toMatchObject({ status: 'running' }); + expect(job?.createdAt).toBeGreaterThan(firstCreatedAt); + + second.release(); + await waitUntil( + () => store.get(scopeId, continued.taskId)?.status === 'completed', + 'second task', + ); + }); + + it('runs without a job when the lever is off', async () => { + const { store, scopeId } = await setup({ generationJobs: false }); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil( + async () => + ( + await methods.getMessages({ + user: userId, + conversationId: threadId, + messageId: `${taskId}:user`, + }) + ).length === 1, + 'the task input row', + ); + + expect(await manager.getJob(threadId)).toBeUndefined(); + + release(); + await waitUntil(() => store.get(scopeId, taskId)?.status === 'completed', 'the task'); + expect((await assistantRow(threadId, taskId))?.subagentTask?.status).toBe('completed'); + }); + + describe('the stored result row', () => { + const textOf = (content: unknown): string => + (content as Array<{ type?: string; text?: string }> | undefined) + ?.filter((part) => part?.type === 'text') + .map((part) => part.text ?? '') + .join('') ?? ''; + + it('carries what a completed child streamed as standard content, beside its text', async () => { + const { store, scopeId } = await setup(); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + + release(); + await waitUntil(() => store.get(scopeId, taskId)?.status === 'completed', 'the task'); + + const row = await assistantRow(threadId, taskId); + expect(row?.text).toBe('Looking into it.'); + expect(textOf(row?.content)).toBe('Looking into it.'); + expect(row?.subagentTask?.status).toBe('completed'); + }); + + it('keeps what a cancelled child had streamed', async () => { + const { store, scopeId } = await setup(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, {}))); + await waitUntil( + async () => + textOf((await manager.getResumeState(threadId))?.aggregatedContent) === + 'Looking into it.', + 'streamed content', + ); + + store.control(scopeId, taskId, { action: 'cancel' }); + await waitUntil( + async () => (await assistantRow(threadId, taskId))?.subagentTask?.status === 'cancelled', + 'the cancelled row', + ); + + const row = await assistantRow(threadId, taskId); + expect(row?.text).toBe('Subagent task was cancelled.'); + expect(textOf(row?.content)).toBe('Looking into it.'); + }); + + it('keeps what a failed child had streamed, with the error flag', async () => { + const { store, scopeId } = await setup(); + const { run, release } = gatedRun('fail'); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + await waitUntil(async () => (await manager.getJob(threadId)) != null, 'the child job'); + + release(); + await waitUntil(async () => (await assistantRow(threadId, taskId)) != null, 'the error row'); + + const row = await assistantRow(threadId, taskId); + expect(row?.error).toBe(true); + expect(textOf(row?.content)).toBe('Looking into it.'); + expect(JSON.stringify(row)).not.toContain('sk-secret'); + }); + + it('leaves the content off when the whole row would outgrow the document', async () => { + const { store, scopeId } = await setup({ generationContentBytes: 8_388_608 }); + /** Stored three times over (text, transcript, content), this passes each field's own + * limit and would exceed MongoDB's 16 MiB document. */ + const big = 'a'.repeat(6 * 1024 * 1024); + const run: SubagentTaskStartRequest['run'] = async (runtime, initialMessages = []) => { + runtime.reportProgress(step()); + runtime.reportProgress(words(big)); + return { + content: big, + messages: [ + ...initialMessages, + new HumanMessage('Investigate the issue.'), + new AIMessage(big), + ], + }; + }; + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + + for (let attempt = 0; attempt < 3000; attempt += 1) { + if (store.get(scopeId, taskId)?.status !== 'running') break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + expect(store.get(scopeId, taskId)?.status).toBe('completed'); + const row = await assistantRow(threadId, taskId); + expect(row?.subagentTask?.status).toBe('completed'); + expect(row?.text).toHaveLength(big.length); + expect(row?.content).toBeUndefined(); + }, 60_000); + + it.each([ + { name: 'the stream is off', activity: { generationJobs: false } }, + { name: 'the content is over its budget', activity: { generationContentBytes: 16 } }, + { name: 'the budget is zero', activity: { generationContentBytes: 0 } }, + ])('stores text only when $name', async ({ activity }) => { + const { store, scopeId } = await setup(activity); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(scopeId, { run }))); + + release(); + await waitUntil(() => store.get(scopeId, taskId)?.status === 'completed', 'the task'); + + const row = await assistantRow(threadId, taskId); + expect(row?.text).toBe('Looking into it.'); + expect(row?.content).toBeUndefined(); + expect(row?.subagentTask?.status).toBe('completed'); + }); + }); + + it('still completes and persists the child when the stream cannot be opened', async () => { + const parentConversationId = randomUUID(); + await methods.saveConvo( + { userId }, + { conversationId: parentConversationId, endpoint: EModelEndpoint.agents, title: 'Parent' }, + ); + const store = new SubagentThreadTaskStore(methods, { + childGenerations: { + open: async () => { + throw new Error('stream store offline'); + }, + }, + }); + const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + const { run, release } = gatedRun(); + const { threadId, taskId } = threadOf(store.start(taskRequest(config.scopeId, { run }))); + + release(); + + await waitUntil(() => store.get(config.scopeId, taskId)?.status === 'completed', 'the task'); + expect((await assistantRow(threadId, taskId))?.subagentTask?.status).toBe('completed'); + }); +}); diff --git a/packages/api/src/agents/subagentThreads.ts b/packages/api/src/agents/subagentThreads.ts index 71532e472b7..05a7fa7d436 100644 --- a/packages/api/src/agents/subagentThreads.ts +++ b/packages/api/src/agents/subagentThreads.ts @@ -38,7 +38,12 @@ import type { ConversationMethods, SubagentTaskResultClaim, } from '@librechat/data-schemas'; -import type { TSubagentActivityConfig, TSteerAppliedEvent, Agents } from 'librechat-data-provider'; +import type { + TSubagentActivityConfig, + TMessageContentParts, + TSteerAppliedEvent, + Agents, +} from 'librechat-data-provider'; import type { BaseMessage, StoredMessage } from '@librechat/agents/langchain/messages'; import type { ScheduledMCPIdentity } from 'librechat-data-provider'; import type { @@ -47,6 +52,11 @@ import type { SubagentActivitySubscription, SubagentActivityTerminalStatus, } from './subagentActivity'; +import type { + ChildGeneration, + ChildGenerationFinal, + ChildGenerationHost, +} from './subagentGeneration'; import type { SubagentTaskControlTransport } from './subagentTaskRouting'; import type { RequestScopedMCPConnectionStore } from '~/mcp/types'; import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; @@ -67,12 +77,12 @@ import { } from './subagentTaskRouting'; import { boundSubagentActivityUpdate, SubagentActivityStream } from './subagentActivity'; import { createSubagentAttemptKey, createSubagentThreadId } from './subagentThreadIds'; +import { SubagentContentJournal, subagentContentFrame } from './subagentContent'; import { describeErrorCauses, describeStackFrames } from './failures/subagent'; import { ActivityRecorder, activityTreeFromProjection } from './digest'; import { runWithDetachedSubagentUsage } from './subagentTaskContext'; import { SUBAGENT_COMPLETION_DELIVERY } from './subagentDelivery'; import { createConcurrencyLimiter } from '~/utils/promise'; -import { SubagentContentJournal } from './subagentContent'; import { retainMCPRequestContext } from '~/mcp/request'; import { getSafeErrorMetadata } from '~/utils/errors'; import { InMemoryEventTransport } from '~/stream'; @@ -80,6 +90,30 @@ import { aggregateEmittedUsage } from './usage'; const SCOPE_VERSION = 1; const DEFAULT_MAX_THREAD_DEPTH = 1; +const CANCELLED_TASK_TEXT = 'Subagent task was cancelled.'; +/** MongoDB refuses a document over 16 MiB. A result row's large fields are budgeted against + * this, leaving room for the rest of the document. */ +const MAX_RESULT_DOCUMENT_BYTES = 15 * 1024 * 1024; + +/** What a child streamed, with its serialized size measured once. */ +type StreamedContent = { parts: TMessageContentParts[]; bytes: number }; + +/** + * The streamed content, when the row can hold it beside the fields it must carry. The content + * is the optional one: a row that would outgrow the document keeps its text, transcript and + * projection and leaves the content off, so a large child never loses its result to it. + */ +function contentWithinDocument( + content: StreamedContent | undefined, + ...siblings: Array +): TMessageContentParts[] | undefined { + if (content == null) return undefined; + let used = content.bytes; + for (const sibling of siblings) { + if (sibling != null) used += Buffer.byteLength(sibling, 'utf8'); + } + return used <= MAX_RESULT_DOCUMENT_BYTES ? content.parts : undefined; +} const DEFAULT_LEASE_TTL_MS = 30_000; const DEFAULT_LEASE_HEARTBEAT_MS = 10_000; const DEFAULT_OWNER_DRAIN_TIMEOUT_MS = 45_000; @@ -165,6 +199,8 @@ interface PreparedThread { /** Stable source-occurrence time shared by first delivery and every replay. */ taskCreatedAt: number; userMessageId?: string; + /** The message the task's input row hangs under: the thread's latest result, or the root. */ + userParentMessageId?: string; replay?: { status: 'completed' | 'error' | 'cancelled'; content: string; @@ -234,6 +270,8 @@ interface TaskThreadLease { /** Resolves only after child persistence and lease cleanup finish. */ execution?: Promise; content?: SubagentContentJournal; + /** The child's resumable stream, when one could be opened. */ + generation?: ChildGeneration; publishAppliedSteer?: (data: Omit, timestamp: string) => void; /** Ordered observational tail; canonical child settlement never awaits it. */ activityTail?: Promise; @@ -284,6 +322,9 @@ async function settleActivityWithin(operation: Promise, timeoutMs: number) export interface SubagentThreadTaskStoreOptions extends InMemorySubagentTaskStoreOptions { activity?: Partial; + /** Opens a resumable generation job for each child run. Without it, or with + * `activity.generationJobs` off, a child has only its activity feed. */ + childGenerations?: ChildGenerationHost; maxThreadDepth?: number; leaseTtlMs?: number; leaseHeartbeatMs?: number; @@ -753,6 +794,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { private readonly cancelUnroutedTask?: SubagentThreadTaskStoreOptions['cancelUnroutedTask']; private readonly onTaskPrepared?: SubagentThreadTaskStoreOptions['onTaskPrepared']; private readonly onTaskSettled?: SubagentThreadTaskStoreOptions['onTaskSettled']; + private readonly childGenerations?: ChildGenerationHost; /** Tasks whose completion wake-up was registered; only they announce settlement. */ private readonly wakeupTaskIds = new Set(); private taskControlTransport?: SubagentTaskControlTransport; @@ -804,6 +846,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { this.cancelUnroutedTask = options.cancelUnroutedTask; this.onTaskPrepared = options.onTaskPrepared; this.onTaskSettled = options.onTaskSettled; + this.childGenerations = options.childGenerations; } /** Receives payload-free authoritative transitions from the SDK task store. */ @@ -1690,6 +1733,12 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { ); } const preparedThread = prepared; + lease.generation = await this.openGeneration( + scope, + request, + runtime.taskId, + preparedThread, + ); const recorder = this.trackActivity(request.scopeId, runtime.taskId); let activitySequence = 0; const append = this.methods.appendSubagentContentBatch; @@ -1764,6 +1813,8 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { forwardProgress(activityEvent); const sequence = activityEvent.activitySequence; lease.content?.record(activityEvent, sequence); + const frame = subagentContentFrame(activityEvent); + if (frame != null) lease.generation?.emit(frame); if (activityEvent.phase === 'steer_applied') { publicRecorder.steer(activityEvent.data as TSteerAppliedEvent); } else { @@ -1890,17 +1941,24 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { } lease.settling = true; await this.closeContent(lease); - await this.persistResult( - scope, - request, - runtime.taskId, - prepared, - result, - detachedUsage, - lease.content?.throughSequence, - appliedSteers, - publicRecorder.snapshot(), - ); + await this.settleGeneration(lease, false, async () => { + const content = await this.persistResult( + scope, + request, + runtime.taskId, + preparedThread, + result, + detachedUsage, + lease.content?.throughSequence, + appliedSteers, + publicRecorder.snapshot(), + this.generationContent(lease), + ); + return this.generationFinal(request, runtime.taskId, preparedThread, { + text: result.content, + content, + }); + }); activityTerminal = 'completed'; return result; } catch (error) { @@ -1915,22 +1973,33 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { const terminalTask = this.get(request.scopeId, runtime.taskId); if (runtime.signal.aborted && terminalTask?.status === 'cancelled') { activityTerminal = 'cancelled'; - if (mayPersist) { - await this.persistCancellation( - scope, - threadId, - request, - runtime.taskId, - detachedUsage, - lease.content?.throughSequence, - publicRecorder.snapshot(), - ).catch((persistError) => { + await this.settleGeneration(lease, true, async () => { + if (!mayPersist) return null; + try { + const written = await this.persistCancellation( + scope, + threadId, + request, + runtime.taskId, + detachedUsage, + lease.content?.throughSequence, + publicRecorder.snapshot(), + this.generationContent(lease), + ); + return written == null + ? null + : this.generationFinal(request, runtime.taskId, prepared, { + text: CANCELLED_TASK_TEXT, + content: written.content, + }); + } catch (persistError) { logger.error( '[subagentThreads] Failed to persist child-thread cancellation', persistError, ); - }); - } + return null; + } + }); throw error; } logger.error('[subagentThreads] Child-thread execution failed', { @@ -1943,23 +2012,35 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { causes: describeErrorCauses(error), frames: describeStackFrames(error), }); - if (mayPersist) { - await this.persistFailure( - scope, - threadId, - request, - runtime.taskId, - error, - detachedUsage, - lease.content?.throughSequence, - publicRecorder.snapshot(), - ).catch((persistError) => { + await this.settleGeneration(lease, false, async () => { + if (!mayPersist) return null; + try { + const written = await this.persistFailure( + scope, + threadId, + request, + runtime.taskId, + error, + detachedUsage, + lease.content?.throughSequence, + publicRecorder.snapshot(), + this.generationContent(lease), + ); + return written == null + ? null + : this.generationFinal(request, runtime.taskId, prepared, { + text: safeErrorMessage(error), + failed: true, + content: written.content, + }); + } catch (persistError) { logger.error( '[subagentThreads] Failed to persist child-thread failure', persistError, ); - }); - } + return null; + } + }); throw new Error(publicFailureDetail(error)); } finally { if (prepared != null) { @@ -1975,6 +2056,10 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { ); } lease.publishAppliedSteer = undefined; + /** A job no path above ended is ended here, before the released lease lets a + * continuation open the thread's next one. Ending it twice is harmless. */ + await lease.generation?.settle(true, async () => null).catch(() => undefined); + lease.generation = undefined; await this.stopAndReleaseSharedLease(scope, threadId, lease); if (this.activeThreads.get(lockKey) === lease) { this.activeThreads.delete(lockKey); @@ -2022,6 +2107,108 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { * Reconstruct only the trusted owner identity carried by the opaque task scope * so tenant-isolated database reads and lazy child initialization do not depend * on request AsyncLocalStorage remaining alive after the parent turn returns. */ + /** Opens the child's resumable stream once its thread and input row are durable. A stream + * that cannot be opened is not an error: the child runs and persists without one. */ + private async openGeneration( + scope: SubagentThreadScope, + request: SubagentTaskStartRequest, + taskId: string, + prepared: PreparedThread, + ): Promise { + if ( + this.childGenerations == null || + !this.activityOptions.generationJobs || + prepared.userMessageId == null + ) { + return undefined; + } + try { + const generation = await this.childGenerations.open({ + userId: scope.userId, + threadId: prepared.conversation.conversationId, + taskId, + parentConversationId: scope.parentConversationId, + ...(prepared.conversation.agent_id == null + ? {} + : { agentId: prepared.conversation.agent_id }), + sender: request.subagentType, + userMessage: { + messageId: prepared.userMessageId, + parentMessageId: prepared.userParentMessageId ?? Constants.NO_PARENT, + text: request.input, + }, + }); + return generation ?? undefined; + } catch (error) { + logger.warn('[subagentThreads] Failed to open the child stream', error); + return undefined; + } + } + + /** Runs the task's own persistence, inside the stream's terminal claim when it has one. */ + private async settleGeneration( + lease: TaskThreadLease, + aborted: boolean, + persist: () => Promise, + ): Promise { + if (lease.generation == null) { + await persist(); + return; + } + await lease.generation.settle(aborted, persist); + } + + /** + * What the child streamed, in the shape every agent message stores, so its result row reads + * like one. Left off when there was no stream, nothing was streamed, or it would outgrow the + * row's budget: the row then keeps only its text and projection, as before. + */ + private generationContent(lease: TaskThreadLease): StreamedContent | undefined { + const limit = this.activityOptions.generationContentBytes; + const parts = lease.generation?.content().filter((part) => part != null); + if (parts == null || parts.length === 0 || limit === 0) return undefined; + try { + const bytes = Buffer.byteLength(JSON.stringify(parts), 'utf8'); + return bytes <= limit ? { parts, bytes } : undefined; + } catch { + return undefined; + } + } + + /** The rows this task just wrote, as the stream's final event names them. */ + private generationFinal( + request: SubagentTaskStartRequest, + taskId: string, + prepared: PreparedThread | undefined, + response: { text: string; failed?: boolean; content?: TMessageContentParts[] }, + ): ChildGenerationFinal | null { + if (prepared == null) return null; + const threadId = prepared.conversation.conversationId; + const userMessageId = `${taskId}:user`; + return { + requestMessage: { + messageId: userMessageId, + parentMessageId: prepared.userParentMessageId ?? Constants.NO_PARENT, + conversationId: threadId, + text: request.input, + sender: 'User', + isCreatedByUser: true, + }, + responseMessage: { + messageId: `${taskId}:assistant`, + parentMessageId: userMessageId, + conversationId: threadId, + sender: request.subagentType, + text: response.text, + content: response.content ?? [], + endpoint: EModelEndpoint.agents, + isCreatedByUser: false, + unfinished: false, + ...(response.failed === true ? { error: true } : {}), + }, + }; + } + private runWithOwnerContext(scope: SubagentThreadScope, run: () => Promise): Promise { return tenantStorage.run( { @@ -3193,7 +3380,11 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { if (task?.status === 'running') { super.control(scopeId, lease.taskId, { action: 'cancel' }); } + return; } + /** The lease proves the child is alive; its stream's idle clock would otherwise + * advance through a long silent tool call. */ + void lease.generation?.touch(); })().finally(() => { if (shared.heartbeatInFlight === renewal) { shared.heartbeatInFlight = undefined; @@ -3596,6 +3787,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { 'The child-thread input has no durable occurrence time.', ), userMessageId, + userParentMessageId: parentMessageId, }; } catch (error) { if (sharedLeaseAcquired) { @@ -3667,7 +3859,8 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { contentThroughSequence?: number, appliedSteers?: readonly AppliedSubagentSteer[], observedActivity?: ReturnType, - ): Promise { + streamed?: StreamedContent, + ): Promise { if (prepared.userMessageId == null) { throw new Error('The child-thread input was not prepared.'); } @@ -3710,6 +3903,12 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { activityProjection.truncated || (!sourceTrusted && activityProjection.activity.length === 0), }; + const content = contentWithinDocument( + streamed, + result.content, + subagentTranscript?.messagesJson, + subagentActivityProjection?.activityJson, + ); const conversation = await this.requireCurrentConversation( scope, request, @@ -3724,6 +3923,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { parentMessageId: prepared.userMessageId, sender: request.subagentType, text: result.content, + ...(content == null ? {} : { content }), endpoint: EModelEndpoint.agents, isCreatedByUser: false, unfinished: false, @@ -3751,6 +3951,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { throw new Error('Unable to persist the child-thread result.'); } await this.touchAfterMessage(scope, conversation.conversationId, taskId, 'completed'); + return content; } private async persistFailure( @@ -3762,12 +3963,16 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { detachedUsage: UsageMetadata[], contentThroughSequence?: number, observedActivity?: ReturnType, - ): Promise { + streamed?: StreamedContent, + ): Promise<{ content?: TMessageContentParts[] } | null> { await this.flushControlReceiptsForSettlement(request.scopeId, taskId); const conversation = await this.currentConversation(scope, request, threadId); if (conversation == null || !(await this.taskInputExists(scope, threadId, taskId))) { - return; + return null; } + const activityJson = JSON.stringify(observedActivity?.activity ?? []); + const text = safeErrorMessage(error); + const content = contentWithinDocument(streamed, text, activityJson); const usage = this.aggregateDetachedUsage(detachedUsage); const savedFailure = await this.methods.saveMessage( { userId: scope.userId, ...retentionContext(conversation) }, @@ -3776,7 +3981,8 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { conversationId: threadId, parentMessageId: `${taskId}:user`, sender: request.subagentType, - text: safeErrorMessage(error), + text, + ...(content == null ? {} : { content }), endpoint: EModelEndpoint.agents, isCreatedByUser: false, unfinished: false, @@ -3789,7 +3995,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { subagentActivityProjection: { taskId, version: 2 as const, - activityJson: JSON.stringify(observedActivity?.activity ?? []), + activityJson, truncated: observedActivity == null || observedActivity.truncated || @@ -3812,6 +4018,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { throw new Error('Unable to persist the child-thread failure.'); } await this.touchAfterMessage(scope, threadId, taskId, 'failed'); + return { ...(content == null ? {} : { content }) }; } private async registerTaskWakeup( @@ -3851,12 +4058,15 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { detachedUsage: UsageMetadata[], contentThroughSequence?: number, observedActivity?: ReturnType, - ): Promise { + streamed?: StreamedContent, + ): Promise<{ content?: TMessageContentParts[] } | null> { await this.flushControlReceiptsForSettlement(request.scopeId, taskId); const conversation = await this.currentConversation(scope, request, threadId); if (conversation == null || !(await this.taskInputExists(scope, threadId, taskId))) { - return; + return null; } + const activityJson = JSON.stringify(observedActivity?.activity ?? []); + const content = contentWithinDocument(streamed, CANCELLED_TASK_TEXT, activityJson); const usage = this.aggregateDetachedUsage(detachedUsage); const savedCancellation = await this.methods.saveMessage( { userId: scope.userId, ...retentionContext(conversation) }, @@ -3865,7 +4075,8 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { conversationId: threadId, parentMessageId: `${taskId}:user`, sender: request.subagentType, - text: 'Subagent task was cancelled.', + text: CANCELLED_TASK_TEXT, + ...(content == null ? {} : { content }), endpoint: EModelEndpoint.agents, isCreatedByUser: false, unfinished: false, @@ -3877,7 +4088,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { subagentActivityProjection: { taskId, version: 2 as const, - activityJson: JSON.stringify(observedActivity?.activity ?? []), + activityJson, truncated: observedActivity == null || observedActivity.truncated || @@ -3900,6 +4111,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { throw new Error('Unable to persist the child-thread cancellation.'); } await this.touchAfterMessage(scope, threadId, taskId, 'cancelled'); + return { ...(content == null ? {} : { content }) }; } private async currentConversation( diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index d3a41986749..ceaf6e5acf9 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -3158,6 +3158,8 @@ class GenerationJobManagerClass { agentEventDetachedActionProducerRequired: jobData.agentEventDetachedActionProducerRequired, agentEventDetachedTerminalEvidence: jobData.agentEventDetachedTerminalEvidence, agentEventBindingId: jobData.agentEventBindingId, + subagentTaskId: jobData.subagentTaskId, + subagentParentConversationId: jobData.subagentParentConversationId, agentEventExpectedAction: jobData.agentEventExpectedAction, agentEventSuspension: jobData.agentEventSuspension, scheduleId: jobData.scheduleId, @@ -8737,6 +8739,24 @@ class GenerationJobManagerClass { await this.waitForProviderExecutionDrain(streamId, job.createdAt, job.providerExecutionId); } + /** + * Keeps a silent run alive. A job's idle clock otherwise advances only with emitted chunks, + * so an owner whose run is working without output, such as a child waiting on a long tool + * call, calls this on its own heartbeat. Only the replica that owns the runtime may touch. + */ + async touchJob(streamId: string, expectedCreatedAt: number): Promise { + const runtime = this.runtimeState.get(streamId); + if ( + !runtime || + runtime.createdAt !== expectedCreatedAt || + !this.isCurrentRuntime(streamId, runtime) || + this.jobStore.touchJob == null + ) { + return false; + } + return this.jobStore.touchJob(streamId, expectedCreatedAt); + } + /** * Set reference to the graph's contentParts array. */ diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts index 2bfcb85b507..72ee6339ba5 100644 --- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts +++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts @@ -3637,6 +3637,69 @@ describe('RedisJobStore Integration Tests', () => { }); }); + describe('touchJob', () => { + test('should refresh the idle clock and lifetimes of a silent running job', async () => { + if (!ioredisClient) { + return; + } + + const { RedisJobStore } = await import('../implementations/RedisJobStore'); + const store = new RedisJobStore(ioredisClient, { runningTtl: 120 }); + await store.initialize(); + + const streamId = `touch-refresh-${Date.now()}`; + const jobKey = `stream:{${streamId}}:job`; + const chunkKey = `stream:{${streamId}}:chunks`; + try { + const job = await store.createJob(streamId, 'user-1', streamId); + await store.appendChunk(streamId, { + event: 'on_message_delta', + data: { id: 'step-1', type: 'text', text: 'first' }, + }); + const staleAt = Date.now() - 600_000; + await ioredisClient.hset(jobKey, 'lastActiveAt', String(staleAt)); + await ioredisClient.expire(jobKey, 5); + await ioredisClient.expire(chunkKey, 5); + + await expect(store.touchJob(streamId, job.createdAt)).resolves.toBe(true); + + expect((await store.getJob(streamId))?.lastActiveAt).toBeGreaterThan(staleAt); + expect(await ioredisClient.ttl(jobKey)).toBeGreaterThan(300); + expect(await ioredisClient.ttl(chunkKey)).toBeGreaterThan(300); + expect(await store.cleanup()).toBe(0); + expect(await store.hasJob(streamId)).toBe(true); + } finally { + await store.destroy(); + } + }); + + test('should refuse another generation and a finished job', async () => { + if (!ioredisClient) { + return; + } + + const { RedisJobStore } = await import('../implementations/RedisJobStore'); + const store = new RedisJobStore(ioredisClient, { runningTtl: 120 }); + await store.initialize(); + + const streamId = `touch-fenced-${Date.now()}`; + const jobKey = `stream:{${streamId}}:job`; + try { + const job = await store.createJob(streamId, 'user-1', streamId); + const before = await ioredisClient.hget(jobKey, 'lastActiveAt'); + + await expect(store.touchJob(streamId, job.createdAt - 1)).resolves.toBe(false); + expect(await ioredisClient.hget(jobKey, 'lastActiveAt')).toBe(before); + + await store.updateJob(streamId, { status: 'complete', completedAt: Date.now() }); + await expect(store.touchJob(streamId, job.createdAt)).resolves.toBe(false); + await expect(store.touchJob(`missing-${Date.now()}`, job.createdAt)).resolves.toBe(false); + } finally { + await store.destroy(); + } + }); + }); + describe('appendChunk TTL Refresh', () => { test('should set TTL on the chunk stream', async () => { if (!ioredisClient) { diff --git a/packages/api/src/stream/__tests__/childGeneration.stream_integration.spec.ts b/packages/api/src/stream/__tests__/childGeneration.stream_integration.spec.ts new file mode 100644 index 00000000000..7ebaad1731d --- /dev/null +++ b/packages/api/src/stream/__tests__/childGeneration.stream_integration.spec.ts @@ -0,0 +1,294 @@ +import { randomUUID } from 'crypto'; +import { StepEvents } from 'librechat-data-provider'; +import type { SubagentContentFrame } from 'librechat-data-provider'; +import type { FinalEvent, ServerSentEvent, StreamSubscription } from '~/types'; +import type { ChildGenerationManager } from '~/agents/subagentGeneration'; +import type { RedisTestClient } from './helpers/redis'; +import { clearRedisTestPrefix, createRedisTestClient } from './helpers/redis'; +import { RedisEventTransport } from '../implementations/RedisEventTransport'; +import { createChildGenerationHost } from '~/agents/subagentGeneration'; +import { GenerationJobManagerClass } from '../GenerationJobManager'; +import { RedisJobStore } from '../implementations/RedisJobStore'; + +jest.spyOn(console, 'log').mockImplementation(); + +const prefix = `child-generation:${randomUUID()}:`; +const managers: GenerationJobManagerClass[] = []; +const subscribers: RedisTestClient[] = []; +const subscriptions: StreamSubscription[] = []; +let redis: RedisTestClient; + +function replica() { + const subscriber = createRedisTestClient(prefix); + subscribers.push(subscriber); + const manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: new RedisJobStore(redis), + eventTransport: new RedisEventTransport(redis, subscriber), + isRedis: true, + cleanupOnComplete: false, + }); + manager.initialize(); + managers.push(manager); + return manager; +} + +const runStep = (id: string): SubagentContentFrame => + ({ + event: StepEvents.ON_RUN_STEP, + data: { + id, + index: 0, + stepDetails: { type: 'message_creation', message_creation: { message_id: id } }, + }, + }) as SubagentContentFrame; + +const text = (id: string, value: string): SubagentContentFrame => + ({ + event: StepEvents.ON_MESSAGE_DELTA, + data: { id, delta: { content: [{ type: 'text', text: value }] } }, + }) as SubagentContentFrame; + +async function waitUntil(condition: () => boolean | Promise, what: string): Promise { + for (let attempt = 0; attempt < 400; attempt += 1) { + if (await condition()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Timed out waiting for ${what}.`); +} + +beforeAll(async () => { + redis = createRedisTestClient(prefix); + await redis.ping(); +}); + +afterEach(async () => { + for (const subscription of subscriptions.splice(0)) subscription.unsubscribe(); + await Promise.all(managers.splice(0).map((manager) => manager.destroy())); + for (const subscriber of subscribers.splice(0)) subscriber.disconnect(); + await clearRedisTestPrefix(redis, prefix); +}); + +afterAll(async () => { + await redis.quit(); +}); + +describe('a background child read from another replica', () => { + test('resumes with everything streamed so far, then follows it to the final event', async () => { + const owner = replica(); + const viewer = replica(); + const threadId = randomUUID(); + const taskId = randomUUID(); + const generation = await createChildGenerationHost(owner).open({ + userId: 'user', + threadId, + taskId, + parentConversationId: randomUUID(), + sender: 'researcher', + userMessage: { messageId: `${taskId}:user`, parentMessageId: 'root', text: 'Look into it' }, + }); + expect(generation).not.toBeNull(); + generation!.emit(runStep('step-1')); + generation!.emit(text('step-1', 'Before the reader ')); + generation!.emit(text('step-1', 'arrived.')); + await waitUntil( + async () => + JSON.stringify( + (await viewer.getResumeState(threadId, generation!.createdAt))?.aggregatedContent ?? [], + ).includes('arrived.'), + 'the durable chunk log', + ); + + const live: ServerSentEvent[] = []; + let final: FinalEvent | undefined; + const resumed = await viewer.subscribeWithResume( + threadId, + (event) => live.push(event), + (event) => { + final = event as FinalEvent; + }, + undefined, + { expectedCreatedAt: generation!.createdAt }, + ); + expect(resumed.subscription).not.toBeNull(); + subscriptions.push(resumed.subscription!); + resumed.subscription!.activate(); + + expect(JSON.stringify(resumed.resumeState?.aggregatedContent)).toContain( + 'Before the reader arrived.', + ); + expect(resumed.resumeState?.userMessage).toMatchObject({ messageId: `${taskId}:user` }); + expect(resumed.resumeState?.responseMessageId).toBe(`${taskId}:assistant`); + expect((await viewer.getJob(threadId))?.metadata.subagentTaskId).toBe(taskId); + await expect(viewer.getActiveJobIdsForUser('user')).resolves.toEqual([]); + + generation!.emit(text('step-1', ' After.')); + await waitUntil(() => JSON.stringify(live).includes(' After.'), 'the live delta'); + expect(JSON.stringify(live)).not.toContain('Before the reader'); + + await generation!.settle(false, async () => ({ + requestMessage: { messageId: `${taskId}:user`, text: 'Look into it' }, + responseMessage: { messageId: `${taskId}:assistant`, text: 'Done' }, + })); + + await waitUntil(() => final != null, 'the final event'); + expect(final).toMatchObject({ + final: true, + conversation: { conversationId: threadId }, + responseMessage: { messageId: `${taskId}:assistant`, text: 'Done' }, + }); + }); + + test('resumes whole after an unwatched child outgrows the early event buffer', async () => { + const owner = replica(); + const viewer = replica(); + const threadId = randomUUID(); + const taskId = randomUUID(); + let written = 0; + let onWritten: (() => void) | undefined; + const counted: ChildGenerationManager = { + getJob: owner.getJob.bind(owner), + createJob: owner.createJob.bind(owner), + touchJob: owner.touchJob.bind(owner), + setContentParts: owner.setContentParts.bind(owner), + claimTerminalJob: owner.claimTerminalJob.bind(owner), + publishTerminalClaim: owner.publishTerminalClaim.bind(owner), + finishTerminalJob: owner.finishTerminalJob.bind(owner), + emitChunk: async (...args) => { + await owner.emitChunk(...args); + written += 1; + onWritten?.(); + }, + }; + const generation = await createChildGenerationHost(counted).open({ + userId: 'user', + threadId, + taskId, + parentConversationId: randomUUID(), + sender: 'researcher', + userMessage: { messageId: `${taskId}:user`, parentMessageId: 'root', text: 'Look into it' }, + }); + /** One write per delta, as at an ordinary token rate: each emit waits for the previous + * write to land, so nothing queues and nothing merges. */ + const emitAndWait = async (frame: SubagentContentFrame) => { + const target = written + 1; + generation!.emit(frame); + while (written < target) { + await new Promise((resolve) => { + onWritten = resolve; + }); + } + }; + await waitUntil(() => written === 1, 'the created event'); + await emitAndWait(runStep('step-1')); + const deltas = 5_200; + for (let index = 0; index < deltas; index += 1) { + await emitAndWait(text('step-1', `${index} `)); + } + expect(written).toBe(deltas + 2); + /** The buffer did overflow, so the resumes below really take the recovery path. */ + expect((await owner.getJob(threadId))?.metadata.earlyBufferOverflow).toBeDefined(); + const expected = Array.from({ length: deltas }, (_, index) => `${index} `).join(''); + const textOf = (content: unknown): string => + (content as Array<{ type?: string; text?: string }> | undefined) + ?.filter((part) => part?.type === 'text') + .map((part) => part.text ?? '') + .join('') ?? ''; + await waitUntil( + async () => + textOf((await viewer.getResumeState(threadId, generation!.createdAt))?.aggregatedContent) + .length === expected.length, + 'the whole chunk log', + ); + + expect((await owner.getJob(threadId))?.status).toBe('running'); + const remote = await viewer.subscribeWithResume( + threadId, + () => undefined, + undefined, + undefined, + { + expectedCreatedAt: generation!.createdAt, + }, + ); + subscriptions.push(remote.subscription!); + const local = await owner.subscribeWithResume(threadId, () => undefined, undefined, undefined, { + expectedCreatedAt: generation!.createdAt, + }); + subscriptions.push(local.subscription!); + + expect(textOf(remote.resumeState?.aggregatedContent)).toBe(expected); + expect(textOf(local.resumeState?.aggregatedContent)).toBe(expected); + expect((await owner.getJob(threadId))?.status).toBe('running'); + }, 120_000); + + test('leaves the child alone when the job store rejects a write, and still ends the job', async () => { + const owner = replica(); + const threadId = randomUUID(); + const taskId = randomUUID(); + const generation = await createChildGenerationHost(owner).open({ + userId: 'user', + threadId, + taskId, + parentConversationId: randomUUID(), + sender: 'researcher', + userMessage: { messageId: `${taskId}:user`, parentMessageId: 'root', text: 'Look into it' }, + }); + generation!.emit(runStep('step-1')); + await waitUntil( + async () => + ((await owner.getResumeState(threadId, generation!.createdAt))?.runSteps ?? []).length === + 1, + 'the first frame', + ); + const append = jest + .spyOn(RedisJobStore.prototype, 'appendChunk') + .mockRejectedValue(new Error('redis append failed')); + try { + expect(() => generation!.emit(text('step-1', 'kept '))).not.toThrow(); + await waitUntil(() => append.mock.calls.length > 0, 'the rejected append'); + generation!.emit(text('step-1', 'whole')); + } finally { + append.mockRestore(); + } + + await waitUntil( + () => JSON.stringify(generation!.content()).includes('kept whole'), + 'the frames written after the failure to fold', + ); + const persist = jest.fn(async () => ({ + requestMessage: { messageId: `${taskId}:user` }, + responseMessage: { messageId: `${taskId}:assistant`, text: 'Done' }, + })); + await expect(generation!.settle(false, persist)).resolves.toBeUndefined(); + expect(persist).toHaveBeenCalledTimes(1); + const status = (await owner.getJob(threadId))?.status; + expect(status === undefined || (status !== 'running' && status !== 'requires_action')).toBe( + true, + ); + }); + + test('keeps a silent child alive across replicas while its owner touches it', async () => { + const owner = replica(); + const viewer = replica(); + const threadId = randomUUID(); + const taskId = randomUUID(); + const generation = await createChildGenerationHost(owner).open({ + userId: 'user', + threadId, + taskId, + parentConversationId: randomUUID(), + sender: 'researcher', + userMessage: { messageId: `${taskId}:user`, parentMessageId: 'root', text: 'Look into it' }, + }); + const jobKey = `stream:{${threadId}}:job`; + const staleAt = Date.now() - 3_600_000; + await waitUntil(async () => (await viewer.getJob(threadId)) != null, 'the job'); + await redis.hset(jobKey, 'lastActiveAt', String(staleAt)); + + await generation!.touch(); + + expect(Number(await redis.hget(jobKey, 'lastActiveAt'))).toBeGreaterThan(staleAt); + expect((await viewer.getJob(threadId))?.status).toBe('running'); + }); +}); diff --git a/packages/api/src/stream/__tests__/childThreadJobs.spec.ts b/packages/api/src/stream/__tests__/childThreadJobs.spec.ts index 4a6fb89beca..6355fae396e 100644 --- a/packages/api/src/stream/__tests__/childThreadJobs.spec.ts +++ b/packages/api/src/stream/__tests__/childThreadJobs.spec.ts @@ -1,7 +1,7 @@ +import { isChildThreadJob, isListedChatJob, isBackgroundChildJob } from '~/stream/metadata'; import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; -import { isChildThreadJob, isListedChatJob } from '~/stream/metadata'; jest.spyOn(console, 'log').mockImplementation(); @@ -58,6 +58,28 @@ describe('child-thread jobs in the active chat list', () => { await expect(manager.getActiveJobIdsForUser(userId)).resolves.toEqual([]); }); + it('leaves a running background child out of the active chat list', async () => { + const backgroundId = 'background-child-thread'; + await manager.createJob(backgroundId, userId, backgroundId, { + initialMetadata: { subagentTaskId: 'task-1', subagentParentConversationId: chatId }, + }); + + await expect(manager.getActiveJobIdsForUser(userId)).resolves.toEqual([chatId]); + const job = await manager.getJob(backgroundId); + expect(job?.metadata).toMatchObject({ + subagentTaskId: 'task-1', + subagentParentConversationId: chatId, + }); + }); + + it('tells a background child apart from an event-bound one', () => { + expect(isBackgroundChildJob({ subagentTaskId: 'task-1' })).toBe(true); + expect(isBackgroundChildJob({ subagentTaskId: '' })).toBe(false); + expect(isBackgroundChildJob({})).toBe(false); + expect(isChildThreadJob({ subagentTaskId: 'task-1' })).toBe(true); + expect(isListedChatJob({ subagentTaskId: 'task-1' })).toBe(false); + }); + it('reads the child-thread marker from the event binding', () => { expect(isChildThreadJob({ agentEventBindingId: 'binding-1' })).toBe(true); expect(isChildThreadJob({ agentEventBindingId: '' })).toBe(false); diff --git a/packages/api/src/stream/__tests__/jobEviction.spec.ts b/packages/api/src/stream/__tests__/jobEviction.spec.ts new file mode 100644 index 00000000000..58de26e1da2 --- /dev/null +++ b/packages/api/src/stream/__tests__/jobEviction.spec.ts @@ -0,0 +1,63 @@ +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; + +jest.spyOn(console, 'log').mockImplementation(); + +describe('InMemoryJobStore eviction at capacity', () => { + let store: InMemoryJobStore; + + beforeEach(() => { + store = new InMemoryJobStore({ ttlAfterComplete: 60000, maxJobs: 2 }); + }); + + afterEach(async () => { + await store.destroy(); + }); + + it('evicts a finished job before an older running one', async () => { + await store.createJob('older-running', 'owner', 'older-running'); + const finished = await store.createJob('newer-finished', 'owner', 'newer-finished'); + await store.updateJob( + 'newer-finished', + { status: 'complete', completedAt: Date.now() }, + finished.createdAt, + ); + + await store.createJob('incoming', 'owner', 'incoming'); + + expect(await store.hasJob('older-running')).toBe(true); + expect(await store.hasJob('newer-finished')).toBe(false); + expect(await store.hasJob('incoming')).toBe(true); + }); + + it.each([ + ['terminal persistence', { terminalPersistencePending: true }], + ['a terminal host action', { terminalHostActionPending: true }], + ['a schedule reconciliation', { preserveForScheduleReconcile: true }], + ['a provider drain', { providerDrained: false }], + ] as const)('does not prefer a terminal job that still owes %s', async (_what, owed) => { + await store.createJob('older-running', 'owner', 'older-running'); + const pending = await store.createJob('newer-pending', 'owner', 'newer-pending'); + await store.updateJob( + 'newer-pending', + { status: 'complete', completedAt: Date.now(), ...owed }, + pending.createdAt, + ); + + await store.createJob('incoming', 'owner', 'incoming'); + + expect(await store.hasJob('newer-pending')).toBe(true); + expect(await store.hasJob('older-running')).toBe(false); + expect(await store.hasJob('incoming')).toBe(true); + }); + + it('still evicts the oldest job when every job is live', async () => { + await store.createJob('oldest', 'owner', 'oldest'); + await store.createJob('newer', 'owner', 'newer'); + + await store.createJob('incoming', 'owner', 'incoming'); + + expect(await store.hasJob('oldest')).toBe(false); + expect(await store.hasJob('newer')).toBe(true); + expect(await store.hasJob('incoming')).toBe(true); + }); +}); diff --git a/packages/api/src/stream/__tests__/jobTouch.spec.ts b/packages/api/src/stream/__tests__/jobTouch.spec.ts new file mode 100644 index 00000000000..e59bdba12e9 --- /dev/null +++ b/packages/api/src/stream/__tests__/jobTouch.spec.ts @@ -0,0 +1,76 @@ +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; + +jest.spyOn(console, 'log').mockImplementation(); + +describe('GenerationJobManager.touchJob', () => { + const streamId = 'silent-run'; + let manager: GenerationJobManagerClass; + let jobStore: InMemoryJobStore; + + beforeEach(() => { + jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000, staleJobTimeout: 1000 }); + manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore, + eventTransport: new InMemoryEventTransport(), + isRedis: false, + cleanupOnComplete: false, + }); + manager.initialize(); + }); + + afterEach(async () => { + await manager.destroy(); + }); + + /** Moves the clock past the stale cutoff, the way a long silent tool call would. */ + const elapse = (ms: number) => { + const base = Date.now(); + return jest.spyOn(Date, 'now').mockImplementation(() => base + ms); + }; + + it('reaps a silent running job that nobody touches', async () => { + await manager.createJob(streamId, 'owner', streamId); + const clock = elapse(5000); + try { + await jobStore.cleanup(); + } finally { + clock.mockRestore(); + } + + expect(await jobStore.hasJob(streamId)).toBe(false); + }); + + it('keeps a silent running job alive once its owner touches it', async () => { + const job = await manager.createJob(streamId, 'owner', streamId); + const clock = elapse(5000); + try { + await expect(manager.touchJob(streamId, job.createdAt)).resolves.toBe(true); + await jobStore.cleanup(); + } finally { + clock.mockRestore(); + } + + expect(await jobStore.hasJob(streamId)).toBe(true); + }); + + it('refuses a touch for another generation of the same stream', async () => { + const job = await manager.createJob(streamId, 'owner', streamId); + + await expect(manager.touchJob(streamId, job.createdAt - 1)).resolves.toBe(false); + }); + + it('refuses a touch once the job has finished', async () => { + const job = await manager.createJob(streamId, 'owner', streamId); + await manager.completeJob(streamId); + + await expect(manager.touchJob(streamId, job.createdAt)).resolves.toBe(false); + await expect(jobStore.touchJob(streamId, job.createdAt)).resolves.toBe(false); + }); + + it('refuses a touch for a stream this manager does not own', async () => { + await expect(manager.touchJob('unknown', Date.now())).resolves.toBe(false); + }); +}); diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts index da09b0ac1dd..87eeb4c7b29 100644 --- a/packages/api/src/stream/implementations/InMemoryJobStore.ts +++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts @@ -1135,6 +1135,15 @@ export class InMemoryJobStore implements IJobStoreV2 { } } + async touchJob(streamId: string, expectedCreatedAt: number): Promise { + const job = this.jobs.get(streamId); + if (job?.createdAt !== expectedCreatedAt || job.status !== 'running') { + return false; + } + this.recordActivity(streamId, expectedCreatedAt); + return true; + } + async hasJob(streamId: string): Promise { return this.jobs.has(streamId); } @@ -1414,16 +1423,34 @@ export class InMemoryJobStore implements IJobStoreV2 { return toDelete.length + retainedRecovered; } + /** Makes room at capacity. A finished job goes before any live one: evicting a running + * or paused generation ends a run somebody is still waiting on. A terminal job that still + * owes persistence, a host action, a schedule reconciliation or a provider drain is not + * finished, since its owner comes back for the record. */ private async evictOldest(): Promise { let oldestId: string | null = null; let oldestTime = Infinity; + let oldestFinishedId: string | null = null; + let oldestFinishedTime = Infinity; for (const [streamId, job] of this.jobs) { if (job.createdAt < oldestTime) { oldestTime = job.createdAt; oldestId = streamId; } + const live = + job.status === 'running' || + job.status === 'requires_action' || + job.terminalPersistencePending === true || + job.terminalHostActionPending === true || + job.preserveForScheduleReconcile === true || + job.providerDrained === false; + if (!live && job.createdAt < oldestFinishedTime) { + oldestFinishedTime = job.createdAt; + oldestFinishedId = streamId; + } } + oldestId = oldestFinishedId ?? oldestId; if (oldestId) { logger.warn(`[InMemoryJobStore] Evicting oldest job: ${oldestId}`); diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts index 37a61a9f225..2f398f9ca60 100644 --- a/packages/api/src/stream/implementations/RedisJobStore.ts +++ b/packages/api/src/stream/implementations/RedisJobStore.ts @@ -983,6 +983,45 @@ const CHUNK_APPEND_LUA = 'redis.call("HSET", KEYS[3], ARGV[4], cjson.encode(receipt)) end ' + 'return 1'; +/** + * The liveness half of CHUNK_APPEND_LUA with no event: an owner whose run is alive but + * silent refreshes the idle clock and every lifetime an append would have extended. Same + * generation and retained-epoch fence; running jobs only, since a paused job keeps its + * own approval window. + * + * KEYS: [chunks, job, steerReceipts, steerReceiptOrder, claimedSteers, steers, + * parkedSteers, generationEpoch] + * ARGV: [runningTtl, expectCreatedAt, nowMs, parkedSteersTtl, generationEpochGraceTtl] + */ +const JOB_TOUCH_LUA = + SCHEDULE_RETENTION_LUA + + 'local currentCreatedAt = redis.call("HGET", KEYS[2], "createdAt") ' + + 'if not currentCreatedAt or currentCreatedAt ~= ARGV[2] then return 0 end ' + + 'if redis.call("HGET", KEYS[2], "status") ~= "running" then return 0 end ' + + 'local retainedEpoch = redis.call("GET", KEYS[8]) ' + + 'if retainedEpoch and retainedEpoch ~= currentCreatedAt then return 0 end ' + + 'local target = tonumber(ARGV[1]) ' + + 'local jobTtl = redis.call("TTL", KEYS[2]) ' + + 'if jobTtl < target then expireScheduleJob(KEYS[2], target) ' + + 'elseif jobTtl > target then target = jobTtl end ' + + 'local recoveryTarget = target ' + + 'if redis.call("HGET", KEYS[2], "recoveredSteerId") then ' + + 'recoveryTarget = target + tonumber(ARGV[4]) ' + + 'local pt = redis.call("TTL", KEYS[7]) ' + + 'if pt >= 0 and pt < recoveryTarget then redis.call("EXPIRE", KEYS[7], recoveryTarget) end end ' + + 'local epochTarget = target + tonumber(ARGV[5]) ' + + 'if retainedEpoch then local epochTtl = redis.call("TTL", KEYS[8]) ' + + 'if epochTtl >= 0 and epochTtl < epochTarget then redis.call("EXPIRE", KEYS[8], epochTarget) end ' + + 'else redis.call("SET", KEYS[8], currentCreatedAt, "EX", epochTarget) end ' + + 'redis.call("HSET", KEYS[2], "lastActiveAt", ARGV[3]) ' + + 'local cur = redis.call("TTL", KEYS[1]) ' + + 'if cur >= 0 and cur < target then redis.call("EXPIRE", KEYS[1], target) end ' + + 'for i = 3, 4 do local rt = redis.call("TTL", KEYS[i]) ' + + 'if rt >= 0 and rt < recoveryTarget then redis.call("EXPIRE", KEYS[i], recoveryTarget) end end ' + + 'for i = 5, 6 do local qt = redis.call("TTL", KEYS[i]) ' + + 'if qt >= 0 and qt < target then redis.call("EXPIRE", KEYS[i], target) end end ' + + 'return 1'; + /** * Batched CHUNK_APPEND_LUA for plain streaming deltas: identical generation/status/epoch * guards and extend-only TTL housekeeping, evaluated once per batch, with one XADD per @@ -5322,6 +5361,27 @@ export class RedisJobStore implements IJobStoreV2 { return appended === 1; } + async touchJob(streamId: string, expectedCreatedAt: number): Promise { + const touched = await this.redis.eval( + JOB_TOUCH_LUA, + 8, + KEYS.chunks(streamId), + KEYS.job(streamId), + KEYS.steerReceipts(streamId), + KEYS.steerReceiptOrder(streamId), + KEYS.claimedSteers(streamId), + KEYS.steers(streamId), + KEYS.parkedSteers(streamId), + KEYS.generationEpoch(streamId), + String(this.runningStorageTtlSeconds()), + String(expectedCreatedAt), + String(Date.now()), + String(this.parkedRecoveryTtlSeconds()), + String(GENERATION_EPOCH_GRACE_TTL_S), + ); + return touched === 1; + } + /** * Buffer a coalescable durable append for the current window. The whole batch * settles together: `true` on commit, `false` under the generation/status @@ -5799,6 +5859,8 @@ export class RedisJobStore implements IJobStoreV2 { ? JSON.parse(data.agentEventDetachedTerminalEvidence) : undefined, agentEventBindingId: data.agentEventBindingId || undefined, + subagentTaskId: data.subagentTaskId || undefined, + subagentParentConversationId: data.subagentParentConversationId || undefined, agentEventExpectedAction: data.agentEventExpectedAction ? JSON.parse(data.agentEventExpectedAction) : undefined, diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index b5768db551e..b7f3b6f70fe 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -368,6 +368,10 @@ export interface SerializableJobData { agentEventDetachedTerminalEvidence?: AgentEventDetachedTerminalEvidence; /** Trusted actor binding copied from the authenticated delivery envelope. */ agentEventBindingId?: string; + /** The background subagent task this generation streams; set only by the task store. */ + subagentTaskId?: string; + /** The conversation that dispatched that task. */ + subagentParentConversationId?: string; agentEventExpectedAction?: import('~/agents/triggers/types').AgentTriggerExpectedAction; /** Versioned pointer to the canonical signed Conversation suspension. */ agentEventSuspension?: import('~/agents/triggers/types').AgentEventSuspensionProjection; @@ -526,6 +530,8 @@ export type JobMetadataPatch = Partial< | 'agentEventDetachedActionProducerRequired' | 'agentEventDetachedTerminalEvidence' | 'agentEventBindingId' + | 'subagentTaskId' + | 'subagentParentConversationId' | 'agentEventExpectedAction' | 'agentEventSuspension' | 'agentEventLegacyTurnToken' @@ -1078,6 +1084,9 @@ export interface IJobStore { clearTerminalHostAction?(streamId: string, expectedCreatedAt?: number): Promise; cleanup(): Promise; recordActivity?(streamId: string, expectedCreatedAt?: number): void; + /** Refreshes a running generation's idle clock and storage lifetimes without an event. + * False when the epoch is not current or the job is not running. */ + touchJob?(streamId: string, expectedCreatedAt: number): Promise; getJobCount(): Promise; getJobCountByStatus(status: JobStatus): Promise; destroy(): Promise; diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.ts index 34e0d206cdc..b41b14bb758 100644 --- a/packages/api/src/stream/metadata.ts +++ b/packages/api/src/stream/metadata.ts @@ -13,17 +13,29 @@ export function parseTerminalPersistenceOwner(value: unknown): 'external' | unde return value; } +type ChildThreadMarkers = Pick; + /** - * A job whose stream is a subagent child thread. An event binding always targets a reserved - * child thread, so its id on the job marks the stream as one. Child threads are not listed - * conversations: a client has no row to mark for them and no record to load. + * A job the subagent task store runs for a background child. The store owns its terminal + * state, so a route that would end or steer the job hands off to the store instead. */ -export function isChildThreadJob(job: Pick): boolean { - return job.agentEventBindingId != null && job.agentEventBindingId !== ''; +export function isBackgroundChildJob(job: Pick): boolean { + return job.subagentTaskId != null && job.subagentTaskId !== ''; +} + +/** + * A job whose stream is a subagent child thread: a background child's, or an event-bound + * child's, since an event binding always targets a reserved child thread. Child threads are + * not listed conversations: a client has no row to mark for them and no record to load. + */ +export function isChildThreadJob(job: ChildThreadMarkers): boolean { + return ( + isBackgroundChildJob(job) || (job.agentEventBindingId != null && job.agentEventBindingId !== '') + ); } /** A job a client can show as a running chat. */ -export function isListedChatJob(job: Pick): boolean { +export function isListedChatJob(job: ChildThreadMarkers): boolean { return !isChildThreadJob(job); } @@ -107,6 +119,12 @@ export function sanitizeJobMetadata(metadata: Partial): J if (metadata.agentEventBindingId) { patch.agentEventBindingId = metadata.agentEventBindingId; } + if (metadata.subagentTaskId) { + patch.subagentTaskId = metadata.subagentTaskId; + } + if (metadata.subagentParentConversationId) { + patch.subagentParentConversationId = metadata.subagentParentConversationId; + } if (metadata.agentEventExpectedAction) { patch.agentEventExpectedAction = metadata.agentEventExpectedAction; } diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts index d8f91abc52e..e1c0fc5cfef 100644 --- a/packages/api/src/types/stream.ts +++ b/packages/api/src/types/stream.ts @@ -75,6 +75,10 @@ export interface GenerationJobMetadata { agentEventDetachedTerminalEvidence?: AgentEventDetachedTerminalEvidence; /** Trusted actor binding copied from the authenticated delivery envelope. */ agentEventBindingId?: string; + /** The background subagent task this generation streams; set only by the task store. */ + subagentTaskId?: string; + /** The conversation that dispatched that task. */ + subagentParentConversationId?: string; /** Optional action evidence contract declared by the authenticated event source. */ agentEventExpectedAction?: AgentTriggerExpectedAction; /** Versioned pointer to the canonical signed suspension stored on the Conversation. */ diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 138a574cef5..3a145b27ea5 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -3117,6 +3117,8 @@ describe('subagent activity policy', () => { endpoints: { agents: { subagentActivity: {} } }, }); expect(parsed.endpoints?.agents?.subagentActivity).toEqual({ + generationJobs: true, + generationContentBytes: 4_194_304, contentRecoveryEnabled: false, contentPageBytes: 262_144, contentFlushMs: 250, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index de32ccec1f7..fb6be8284b6 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1861,6 +1861,12 @@ export type HostFileEditLimits = z.infer; /** Server-side resource and recovery policy for ephemeral child activity. */ export const subagentActivityConfigSchema = z.object({ + /** Run each background child as a resumable generation job keyed by its thread id, so its + * thread streams and resumes the way a chat does. Off leaves only the activity feed. */ + generationJobs: z.boolean().default(true), + /** Largest content, in serialized bytes, stored on a child's result row beside its text. + * Larger content is left off so the row always fits; 0 stores none. */ + generationContentBytes: z.number().int().min(0).max(8_388_608).default(4_194_304), contentRecoveryEnabled: z.boolean().default(false), /** Batch public content writes independently of the lossy live activity projection. */ contentPageBytes: z.number().int().min(1024).max(4_194_304).default(262_144),