Skip to content

Commit bc01093

Browse files
vercel-ai-sdk[bot]cipher416aayush-kapoorvercel[bot]
authored
Backport: fix(openai): support file-url parts in tool output content (#13933)
This is an automated backport of #13663 to the release-v6.0 branch. FYI @cipher416 Co-authored-by: Cristoper Anderson <67546516+cipher416@users.noreply.github.com> Co-authored-by: Aayush Kapoor <aayushkapoor34@gmail.com> Co-authored-by: Aayush Kapoor <83492835+aayush-kapoor@users.noreply.github.com> Co-authored-by: vercel[bot] <35613825+vercel[bot]@users.noreply.github.com>
1 parent 2fa944a commit bc01093

5 files changed

Lines changed: 211 additions & 0 deletions

File tree

‎.changeset/healthy-toes-roll.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@ai-sdk/openai": patch
3+
---
4+
5+
fix(openai): support file-url parts in tool output content

‎examples/ai-functions/src/generate-text/openai/responses-file-url-tool-result.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import { openai } from '@ai-sdk/openai';
2+
import { generateText, stepCountIs, tool } from 'ai';
3+
import { z } from 'zod';
4+
import { run } from '../../lib/run';
5+
6+
run(async () => {
7+
const readPDFDocument = tool({
8+
description: `Read and return a PDF document by URL`,
9+
inputSchema: z.object({}),
10+
execute: async () => ({
11+
success: true,
12+
description: 'Successfully loaded PDF document',
13+
pdfUrl: 'https://www.berkshirehathaway.com/letters/2024ltr.pdf',
14+
}),
15+
toModelOutput({ output }) {
16+
return {
17+
type: 'content',
18+
value: [
19+
{
20+
type: 'text',
21+
text: output.description,
22+
},
23+
{
24+
type: 'file-url',
25+
url: output.pdfUrl,
26+
},
27+
],
28+
};
29+
},
30+
});
31+
32+
const result = await generateText({
33+
model: openai.responses('gpt-4.1-mini'),
34+
prompt:
35+
'Please read the PDF document using the tool provided and return a summary of it.',
36+
tools: {
37+
readPDFDocument,
38+
},
39+
stopWhen: stepCountIs(4),
40+
});
41+
42+
console.log(`Assistant response: ${JSON.stringify(result.text, null, 2)}`);
43+
console.log(`Warnings: ${JSON.stringify(result.warnings, null, 2)}`);
44+
});

‎packages/openai/src/responses/convert-to-openai-responses-input.test.ts‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2214,6 +2214,106 @@ describe('convertToOpenAIResponsesInput', () => {
22142214
`);
22152215
});
22162216

2217+
it('should convert single tool result part with multipart that contains file-url', async () => {
2218+
const result = await convertToOpenAIResponsesInput({
2219+
toolNameMapping: testToolNameMapping,
2220+
prompt: [
2221+
{
2222+
role: 'tool',
2223+
content: [
2224+
{
2225+
type: 'tool-result',
2226+
toolCallId: 'call_123',
2227+
toolName: 'search',
2228+
output: {
2229+
type: 'content',
2230+
value: [
2231+
{
2232+
type: 'file-url',
2233+
url: 'https://example.com/document.pdf',
2234+
},
2235+
],
2236+
},
2237+
},
2238+
],
2239+
},
2240+
],
2241+
systemMessageMode: 'system',
2242+
providerOptionsName: 'openai',
2243+
store: true,
2244+
});
2245+
2246+
expect(result.input).toMatchInlineSnapshot(`
2247+
[
2248+
{
2249+
"call_id": "call_123",
2250+
"output": [
2251+
{
2252+
"file_url": "https://example.com/document.pdf",
2253+
"type": "input_file",
2254+
},
2255+
],
2256+
"type": "function_call_output",
2257+
},
2258+
]
2259+
`);
2260+
expect(result.warnings).toEqual([]);
2261+
});
2262+
2263+
it('should convert single tool result part with multipart with mixed content including file-url', async () => {
2264+
const result = await convertToOpenAIResponsesInput({
2265+
toolNameMapping: testToolNameMapping,
2266+
prompt: [
2267+
{
2268+
role: 'tool',
2269+
content: [
2270+
{
2271+
type: 'tool-result',
2272+
toolCallId: 'call_123',
2273+
toolName: 'search',
2274+
output: {
2275+
type: 'content',
2276+
value: [
2277+
{
2278+
type: 'text',
2279+
text: 'Here is the file you asked for:',
2280+
},
2281+
{
2282+
type: 'file-url',
2283+
url: 'https://example.com/test.pdf',
2284+
},
2285+
],
2286+
},
2287+
},
2288+
],
2289+
},
2290+
],
2291+
systemMessageMode: 'system',
2292+
providerOptionsName: 'openai',
2293+
store: true,
2294+
});
2295+
2296+
expect(result.input).toMatchInlineSnapshot(`
2297+
[
2298+
{
2299+
"call_id": "call_123",
2300+
"output": [
2301+
{
2302+
"text": "Here is the file you asked for:",
2303+
"type": "input_text",
2304+
},
2305+
{
2306+
"file_url": "https://example.com/test.pdf",
2307+
"type": "input_file",
2308+
},
2309+
],
2310+
"type": "function_call_output",
2311+
},
2312+
]
2313+
`);
2314+
expect(result.warnings).toEqual([]);
2315+
});
2316+
22172317
it('should convert single tool result part with multipart with mixed content (text, image, file)', async () => {
22182318
const base64Data = 'AQIDBAU=';
22192319
const result = await convertToOpenAIResponsesInput({
@@ -4023,6 +4123,55 @@ describe('convertToOpenAIResponsesInput', () => {
40234123
`);
40244124
});
40254125

4126+
it('should convert custom tool result content output with file-url', async () => {
4127+
const result = await convertToOpenAIResponsesInput({
4128+
toolNameMapping: testToolNameMapping,
4129+
prompt: [
4130+
{
4131+
role: 'tool',
4132+
content: [
4133+
{
4134+
type: 'tool-result',
4135+
toolCallId: 'call_custom_006',
4136+
toolName: 'write_sql',
4137+
output: {
4138+
type: 'content',
4139+
value: [
4140+
{ type: 'text', text: 'Here is the file:' },
4141+
{ type: 'file-url', url: 'https://example.com/test.pdf' },
4142+
],
4143+
},
4144+
},
4145+
],
4146+
},
4147+
],
4148+
systemMessageMode: 'system',
4149+
providerOptionsName: 'openai',
4150+
store: true,
4151+
customProviderToolNames,
4152+
});
4153+
4154+
expect(result.input).toMatchInlineSnapshot(`
4155+
[
4156+
{
4157+
"call_id": "call_custom_006",
4158+
"output": [
4159+
{
4160+
"text": "Here is the file:",
4161+
"type": "input_text",
4162+
},
4163+
{
4164+
"file_url": "https://example.com/test.pdf",
4165+
"type": "input_file",
4166+
},
4167+
],
4168+
"type": "custom_tool_call_output",
4169+
},
4170+
]
4171+
`);
4172+
expect(result.warnings).toEqual([]);
4173+
});
4174+
40264175
it('should not emit custom_tool_call when customProviderToolNames is not provided', async () => {
40274176
const result = await convertToOpenAIResponsesInput({
40284177
toolNameMapping: testToolNameMapping,

‎packages/openai/src/responses/convert-to-openai-responses-input.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -709,6 +709,11 @@ export async function convertToOpenAIResponsesInput({
709709
filename: item.filename ?? 'data',
710710
file_data: `data:${item.mediaType};base64,${item.data}`,
711711
};
712+
case 'file-url':
713+
return {
714+
type: 'input_file' as const,
715+
file_url: item.url,
716+
};
712717
default:
713718
warnings.push({
714719
type: 'other',
@@ -773,6 +778,13 @@ export async function convertToOpenAIResponsesInput({
773778
};
774779
}
775780

781+
case 'file-url': {
782+
return {
783+
type: 'input_file' as const,
784+
file_url: item.url,
785+
};
786+
}
787+
776788
default: {
777789
warnings.push({
778790
type: 'other',

‎packages/openai/src/responses/openai-responses-api.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ export type OpenAIResponsesFunctionCallOutput = {
106106
| { type: 'input_text'; text: string }
107107
| { type: 'input_image'; image_url: string }
108108
| { type: 'input_file'; filename: string; file_data: string }
109+
| { type: 'input_file'; file_url: string }
109110
>;
110111
};
111112

0 commit comments

Comments
 (0)