diff --git a/nodes/VlmRun/ApiService.ts b/nodes/VlmRun/ApiService.ts index f99420a..d70ce17 100644 --- a/nodes/VlmRun/ApiService.ts +++ b/nodes/VlmRun/ApiService.ts @@ -1,5 +1,5 @@ import { IExecuteFunctions, IDataObject, ILoadOptionsFunctions } from 'n8n-workflow'; -import { FileRequest, PredictionResponse, FileResponse, ImageRequest } from './types'; +import { FileRequest, PredictionResponse, FileResponse, ImageRequest, ChatMessage, ChatCompletionRequest, ResponseFormat } from './types'; import { AgentCreateRequest, VlmRunClient } from './VlmRunClient'; import { FileService, PredictionService, DomainService } from './services'; @@ -155,4 +155,23 @@ export class ApiService { return { url: response.public_url, ...response }; } + + // Chat Completion Operations + static async chatCompletion( + ef: IExecuteFunctions, + messages: ChatMessage[], + model: string, + max_tokens?: number, + response_format?: ResponseFormat, + ): Promise { + const client = await this.initializeVlmRun(ef); + const request: ChatCompletionRequest = { messages, model }; + if (max_tokens !== undefined) { + request.max_tokens = max_tokens; + } + if (response_format !== undefined) { + request.response_format = response_format; + } + return client.chat.completions(request); + } } diff --git a/nodes/VlmRun/VlmRun.node.ts b/nodes/VlmRun/VlmRun.node.ts index 04268e5..94523af 100644 --- a/nodes/VlmRun/VlmRun.node.ts +++ b/nodes/VlmRun/VlmRun.node.ts @@ -9,7 +9,7 @@ import { INodePropertyOptions, INodeTypeDescription, } from 'n8n-workflow'; -import { FileRequest } from './types'; +import { FileRequest, ChatMessage } from './types'; import { ApiService } from './ApiService'; import { processFile, processImageRequest } from './utils'; @@ -74,15 +74,21 @@ export class VlmRun implements INodeType { description: 'Execute an agent', action: 'Execute agent', }, - { - name: 'Manage Files', - value: 'file', - description: 'List uploaded files or upload new files to VLM Run', - action: 'Manage files', - }, - ], - default: 'document', - }, + { + name: 'Manage Files', + value: 'file', + description: 'List uploaded files or upload new files to VLM Run', + action: 'Manage files', + }, + { + name: 'Chat Completion', + value: 'chatCompletion', + description: 'Generate chat completions using OpenAI-compatible API', + action: 'Chat completion', + }, + ], + default: 'document', + }, // File field for document, image, audio, video operations { displayName: 'File', @@ -281,6 +287,252 @@ export class VlmRun implements INodeType { required: true, description: 'URL to call when processing is complete', }, + // Chat Completion Properties + { + displayName: 'Model', + name: 'model', + type: 'options', + displayOptions: { + show: { + operation: ['chatCompletion'], + }, + }, + options: [ + { + name: 'vlmrun-orion-1:fast', + value: 'vlmrun-orion-1:fast', + }, + { + name: 'vlmrun-orion-1:auto', + value: 'vlmrun-orion-1:auto', + }, + { + name: 'vlmrun-orion-1:pro', + value: 'vlmrun-orion-1:pro', + }, + ], + default: 'vlmrun-orion-1:auto', + required: true, + description: 'Model to use for chat completion', + }, + { + displayName: 'Prompt', + name: 'prompt', + type: 'fixedCollection', + typeOptions: { + sortable: true, + multipleValues: true, + }, + displayOptions: { + show: { + operation: ['chatCompletion'], + }, + }, + placeholder: 'Add Message', + default: {}, + options: [ + { + displayName: 'Messages', + name: 'messages', + values: [ + { + displayName: 'Role', + name: 'role', + type: 'options', + options: [ + { + name: 'Assistant', + value: 'assistant', + }, + { + name: 'System', + value: 'system', + }, + { + name: 'User', + value: 'user', + }, + ], + default: 'user', + }, + { + displayName: 'Content', + name: 'content', + type: 'string', + default: '', + }, + ], + }, + ], + }, + { + displayName: 'Input Type', + name: 'inputType', + type: 'options', + displayOptions: { + show: { + operation: ['chatCompletion'], + }, + }, + options: [ + { + name: 'Image URL(s)', + value: 'image', + }, + { + name: 'Video URL(s)', + value: 'video', + }, + { + name: 'File URL(s)', + value: 'file', + }, + ], + default: 'image', + description: 'Type of media input', + }, + { + displayName: 'Image URL(s)', + name: 'imageUrls', + type: 'fixedCollection', + placeholder: 'Add URL', + typeOptions: { + multipleValues: true, + }, + displayOptions: { + show: { + operation: ['chatCompletion'], + inputType: ['image'], + }, + }, + default: {}, + description: 'Image URL(s) to include in the chat completion', + options: [ + { + displayName: 'URL', + name: 'url', + values: [ + { + displayName: 'URL', + name: 'url', + type: 'string', + default: '', + placeholder: 'https://example.com/image.jpg', + required: true, + description: 'Image URL', + }, + ], + }, + ], + }, + { + displayName: 'Video URL(s)', + name: 'videoUrls', + type: 'fixedCollection', + placeholder: 'Add URL', + typeOptions: { + multipleValues: true, + }, + displayOptions: { + show: { + operation: ['chatCompletion'], + inputType: ['video'], + }, + }, + default: {}, + description: 'Video URL(s) to include in the chat completion', + options: [ + { + displayName: 'URL', + name: 'url', + values: [ + { + displayName: 'URL', + name: 'url', + type: 'string', + default: '', + placeholder: 'https://example.com/video.mp4', + required: true, + description: 'Video URL', + }, + ], + }, + ], + }, + { + displayName: 'File URL(s)', + name: 'fileUrls', + type: 'fixedCollection', + placeholder: 'Add URL', + typeOptions: { + multipleValues: true, + }, + displayOptions: { + show: { + operation: ['chatCompletion'], + inputType: ['file'], + }, + }, + default: {}, + description: 'File URL(s) to include in the chat completion (PDFs, documents, etc.)', + options: [ + { + displayName: 'URL', + name: 'url', + values: [ + { + displayName: 'URL', + name: 'url', + type: 'string', + default: '', + placeholder: 'https://example.com/document.pdf', + required: true, + description: 'File URL', + }, + ], + }, + ], + }, + { + displayName: 'Simplify Output', + name: 'simplifyOutput', + type: 'boolean', + displayOptions: { + show: { + operation: ['chatCompletion'], + }, + }, + default: false, + description: 'Whether to return only the message content instead of full API response', + }, + // { + // displayName: 'Max Tokens', + // name: 'maxTokens', + // type: 'number', + // displayOptions: { + // show: { + // operation: ['chatCompletion'], + // }, + // }, + // default: 32768, + // typeOptions: { + // maxValue: 32768, + // }, + // description: 'The maximum number of tokens to generate in the completion', + // }, + // { + // displayName: 'Response Format', + // name: 'responseFormat', + // type: 'json', + // displayOptions: { + // show: { + // operation: ['chatCompletion'], + // }, + // }, + // default: '', + // description: + // 'Specify the format of the response. Use JSON schema format: {"type": "json_schema", "schema": {...}}', + // }, ], }; @@ -309,7 +561,7 @@ export class VlmRun implements INodeType { case 'document': case 'audio': case 'video': { - const model = 'vlm-1'; // Use hardcoded model value + const model = 'vlmrun-orion-1:auto'; // Use hardcoded model value const file = this.getNodeParameter('file', i) as string; const { buffer, fileName } = await processFile(this, items[i], i, file); const domain = this.getNodeParameter('domain', 0) as string; @@ -418,6 +670,209 @@ export class VlmRun implements INodeType { break; } + case 'chatCompletion': { + const promptParam = this.getNodeParameter('prompt', i) as IDataObject; + const model = this.getNodeParameter('model', i) as string; + const inputType = this.getNodeParameter('inputType', i) as string; + const simplifyOutput = this.getNodeParameter('simplifyOutput', i) as boolean; + // const maxTokens = this.getNodeParameter('maxTokens', i) as number | undefined; + // const responseFormatParam = this.getNodeParameter('responseFormat', i) as string | undefined; + + // Extract messages from fixedCollection + const messagesData = (promptParam.messages as IDataObject[]) || []; + + // Process image URLs + const imageUrls: string[] = []; + if (inputType === 'image') { + const imageUrlsParam = this.getNodeParameter('imageUrls', i) as IDataObject; + if (imageUrlsParam && imageUrlsParam.url) { + const urlEntries = Array.isArray(imageUrlsParam.url) ? imageUrlsParam.url : [imageUrlsParam.url]; + for (const entry of urlEntries) { + if (entry && typeof entry === 'object' && entry.url) { + const url = entry.url as string; + if (url && url.trim()) { + imageUrls.push(url.trim()); + } + } + } + } + } + + // Process video URLs + const videoUrls: string[] = []; + if (inputType === 'video') { + const videoUrlsParam = this.getNodeParameter('videoUrls', i) as IDataObject; + if (videoUrlsParam && videoUrlsParam.url) { + const urlEntries = Array.isArray(videoUrlsParam.url) ? videoUrlsParam.url : [videoUrlsParam.url]; + for (const entry of urlEntries) { + if (entry && typeof entry === 'object' && entry.url) { + const url = entry.url as string; + if (url && url.trim()) { + videoUrls.push(url.trim()); + } + } + } + } + } + + // Process file URLs + const fileUrls: string[] = []; + if (inputType === 'file') { + const fileUrlsParam = this.getNodeParameter('fileUrls', i) as IDataObject; + if (fileUrlsParam && fileUrlsParam.url) { + const urlEntries = Array.isArray(fileUrlsParam.url) ? fileUrlsParam.url : [fileUrlsParam.url]; + for (const entry of urlEntries) { + if (entry && typeof entry === 'object' && entry.url) { + const url = entry.url as string; + if (url && url.trim()) { + fileUrls.push(url.trim()); + } + } + } + } + } + + // Build messages with support for images, videos, and files + const messages: ChatMessage[] = messagesData.map((msg: IDataObject, index: number) => { + const role = msg.role as string; + const content = msg.content as string; + + if ((imageUrls.length > 0 || videoUrls.length > 0 || fileUrls.length > 0) && + role === 'user' && + index === messagesData.length - 1) { + // Create content array with text, images, videos, and files + const contentParts: Array<{ + type: 'text' | 'image_url' | 'video_url' | 'file_url'; + text?: string; + image_url?: { url: string }; + video_url?: { url: string }; + file_url?: { url: string }; + }> = []; + + if (content && content.trim()) { + contentParts.push({ + type: 'text', + text: content, + }); + } + + // Add all images + for (const imageUrl of imageUrls) { + contentParts.push({ + type: 'image_url', + image_url: { + url: imageUrl, + }, + }); + } + + // Add all videos + for (const videoUrl of videoUrls) { + contentParts.push({ + type: 'video_url', + video_url: { + url: videoUrl, + }, + }); + } + + // Add all files + for (const fileUrl of fileUrls) { + contentParts.push({ + type: 'file_url', + file_url: { + url: fileUrl, + }, + }); + } + + return { + role, + content: contentParts, + }; + } + + // Otherwise, return simple text content + return { + role, + content, + }; + }); + + // Validate messages + if (messages.length === 0) { + throw new NodeOperationError( + this.getNode(), + 'At least one message is required. Please add a message to the prompt.', + ); + } + + // Validate message structure + for (const message of messages) { + if (!message || !message.role) { + throw new NodeOperationError( + this.getNode(), + 'Each message must have a "role" property.', + ); + } + + if (typeof message.content === 'string') { + if (!message.content) { + throw new NodeOperationError( + this.getNode(), + 'Text message content cannot be empty.', + ); + } + } else if (Array.isArray(message.content)) { + if (message.content.length === 0) { + throw new NodeOperationError( + this.getNode(), + 'Message content array cannot be empty.', + ); + } + } else { + throw new NodeOperationError( + this.getNode(), + 'Message content must be either a string or an array of content parts.', + ); + } + } + + // Parse response_format if provided + // let responseFormat: { type: string; schema?: any } | undefined; + // if (responseFormatParam && responseFormatParam !== '') { + // try { + // responseFormat = JSON.parse(responseFormatParam); + // } catch (error) { + // throw new NodeOperationError( + // this.getNode(), + // 'Invalid JSON format for response_format. Please provide a valid JSON object.', + // ); + // } + // } + + response = await ApiService.chatCompletion(this, messages, model); + + // Simplify output if requested + if (simplifyOutput && response && (response as any).choices && (response as any).choices.length > 0) { + const choices = (response as any).choices; + // Return simplified structure matching OpenAI format + response = choices.map((choice: any, index: number) => ({ + index: choice.index !== undefined ? choice.index : index, + message: { + role: choice.message?.role || 'assistant', + content: choice.message?.content || '', + refusal: choice.message?.refusal || null, + annotations: choice.message?.annotations || [], + }, + logprobs: choice.logprobs || null, + finish_reason: choice.finish_reason || 'stop', + })); + } + + break; + } + default: throw new NodeOperationError( this.getNode(), diff --git a/nodes/VlmRun/VlmRunClient.ts b/nodes/VlmRun/VlmRunClient.ts index 08524ef..15a9cce 100644 --- a/nodes/VlmRun/VlmRunClient.ts +++ b/nodes/VlmRun/VlmRunClient.ts @@ -1,4 +1,5 @@ import { IExecuteFunctions, IHttpRequestOptions, IHttpRequestMethods } from 'n8n-workflow'; +import { ChatCompletionRequest } from './types'; import packageJson from '../../package.json'; export interface VlmRunConfig { @@ -150,6 +151,15 @@ export class VlmRunClient { errorDetail = error.response.body || error.message; } } + // Log full error details for debugging + console.error('VlmRun API Error:', { + url, + method, + status: error.response?.status, + statusText: error.response?.statusText, + body: error.response?.body, + errorDetail, + }); throw new Error(`HTTP ${error.response?.status || 'Error'}: ${errorDetail}`); } } @@ -225,7 +235,7 @@ export class VlmRunClient { if (request.file instanceof Buffer) { // Convert Buffer to File-like object - const blob = new Blob([request.file]); + const blob = new Blob([new Uint8Array(request.file)]); formData.append('file', blob, 'uploaded-file'); } else { // request.file is a File object @@ -342,6 +352,14 @@ export class VlmRunClient { }, }; + // Chat Completion API + public chat = { + completions: async (request: ChatCompletionRequest): Promise => { + const endpoint = '/openai/chat/completions'; + return this.makeAgentRequest('POST', endpoint, request); + }, + }; + // Agent API public agent = { get: async (): Promise => { diff --git a/nodes/VlmRun/types.ts b/nodes/VlmRun/types.ts index 1318e7e..b1b4dcb 100644 --- a/nodes/VlmRun/types.ts +++ b/nodes/VlmRun/types.ts @@ -33,3 +33,32 @@ export interface FileResponse extends IDataObject { created_at: string; object?: string; } + +export interface ChatMessage { + role: string; + content: string | Array<{ + type: 'text' | 'image_url' | 'video_url' | 'file_url'; + text?: string; + image_url?: { + url: string; + }; + video_url?: { + url: string; + }; + file_url?: { + url: string; + }; + }>; +} + +export interface ResponseFormat { + type: 'json_schema' | string; + schema?: IDataObject; +} + +export interface ChatCompletionRequest { + messages: ChatMessage[]; + model: string; + max_tokens?: number; + response_format?: ResponseFormat; +} \ No newline at end of file