Skip to content

Commit fafc3f2

Browse files
lgrammellgrammel
authored
chore (ai): change file to parts to use urls instead of data (vercel#6068)
## Background File UI parts currently have a data property with base64 encoded data. However, this makes it harder to use file parts in the client where often urls are desired (for images, links) and also prevents sending file parts with URLs . ## Summary Change `data` property in file UI parts to `url` and send data URLs. ## Future Work Replace attachments with file ui parts.
1 parent fa8c550 commit fafc3f2

File tree

15 files changed

+48
-42
lines changed

15 files changed

+48
-42
lines changed

‎.changeset/proud-cows-bathe.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'ai': major
3+
---
4+
5+
chore (ai): change file to parts to use urls instead of data

‎examples/next-openai/app/use-chat-image-output/page.tsx

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,8 @@ export default function Chat() {
2020
part.mediaType.startsWith('image/')
2121
) {
2222
return (
23-
// eslint-disable-next-line @next/next/no-img-element, jsx-a11y/alt-text
24-
<img
25-
key={index}
26-
src={`data:${part.mediaType};base64,${part.data}`}
27-
/>
23+
// eslint-disable-next-line @next/next/no-img-element
24+
<img key={index} src={part.url} alt="Generated image" />
2825
);
2926
}
3027
})}

‎examples/next-openai/app/use-chat-persistence-single-message-image-output/[id]/chat.tsx

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,8 @@ export default function Chat({
3333
part.mediaType.startsWith('image/')
3434
) {
3535
return (
36-
// eslint-disable-next-line @next/next/no-img-element, jsx-a11y/alt-text
37-
<img
38-
key={index}
39-
src={`data:${part.mediaType};base64,${part.data}`}
40-
/>
36+
// eslint-disable-next-line @next/next/no-img-element
37+
<img key={index} src={part.url} alt="Generated image" />
4138
);
4239
}
4340
})}

‎packages/ai/core/generate-text/__snapshots__/stream-text.test.ts.snap

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4028,11 +4028,11 @@ exports[`streamText > result.pipeDataStreamToResponse > should write file conten
40284028
[
40294029
"f:{"messageId":"msg-0"}
40304030
",
4031-
"k:{"mimeType":"text/plain","data":"Hello World"}
4031+
"k:{"mimeType":"text/plain","url":"data:text/plain;base64,Hello World"}
40324032
",
40334033
"0:"Hello!"
40344034
",
4035-
"k:{"mimeType":"image/jpeg","data":"QkFVRw=="}
4035+
"k:{"mimeType":"image/jpeg","url":"data:image/jpeg;base64,QkFVRw=="}
40364036
",
40374037
"e:{"finishReason":"stop","usage":{"promptTokens":3,"completionTokens":10},"isContinued":false}
40384038
",
@@ -4525,11 +4525,11 @@ exports[`streamText > result.toDataStream > should send file content 1`] = `
45254525
[
45264526
"f:{"messageId":"msg-0"}
45274527
",
4528-
"k:{"mimeType":"text/plain","data":"Hello World"}
4528+
"k:{"mimeType":"text/plain","url":"data:text/plain;base64,Hello World"}
45294529
",
45304530
"0:"Hello!"
45314531
",
4532-
"k:{"mimeType":"image/jpeg","data":"QkFVRw=="}
4532+
"k:{"mimeType":"image/jpeg","url":"data:image/jpeg;base64,QkFVRw=="}
45334533
",
45344534
"e:{"finishReason":"stop","usage":{"promptTokens":3,"completionTokens":10},"isContinued":false}
45354535
",

‎packages/ai/core/generate-text/stream-text.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1612,8 +1612,8 @@ However, the LLM results are expected to be small enough to not cause issues.
16121612
controller.enqueue(
16131613
// TODO update protocol to v2 or replace with event stream
16141614
formatDataStreamPart('file', {
1615-
mimeType: chunk.file.mediaType,
1616-
data: chunk.file.base64,
1615+
mimeType: chunk.file.mediaType, // TODO mediaType
1616+
url: `data:${chunk.file.mediaType};base64,${chunk.file.base64}`,
16171617
}),
16181618
);
16191619
break;

‎packages/ai/core/prompt/__snapshots__/append-response-messages.test.ts.snap

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,9 +302,9 @@ exports[`appendResponseMessages > after user message > appends assistant message
302302
"type": "step-start",
303303
},
304304
{
305-
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII=",
306305
"mediaType": "image/png",
307306
"type": "file",
307+
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII=",
308308
},
309309
],
310310
"role": "assistant",

‎packages/ai/core/prompt/append-response-messages.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ Internal. For test use only. May change without notice.
116116
parts.push({
117117
type: 'file' as const,
118118
mediaType: part.mediaType ?? part.mimeType,
119-
data: convertDataContentToBase64String(part.data),
119+
url: `data:${part.mediaType ?? part.mimeType};base64,${convertDataContentToBase64String(part.data)}`,
120120
});
121121
break;
122122
}

‎packages/ai/core/prompt/convert-to-core-messages.test.ts

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ describe('convertToCoreMessages', () => {
306306
{
307307
type: 'file',
308308
mediaType: 'image/png',
309-
data: 'dGVzdA==',
309+
url: 'data:image/png;base64,dGVzdA==',
310310
},
311311
],
312312
},
@@ -315,7 +315,13 @@ describe('convertToCoreMessages', () => {
315315
expect(result).toEqual([
316316
{
317317
role: 'assistant',
318-
content: [{ type: 'file', mediaType: 'image/png', data: 'dGVzdA==' }],
318+
content: [
319+
{
320+
type: 'file',
321+
mediaType: 'image/png',
322+
data: 'data:image/png;base64,dGVzdA==',
323+
},
324+
],
319325
},
320326
] satisfies CoreMessage[]);
321327
});

‎packages/ai/core/prompt/convert-to-core-messages.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ export function convertToCoreMessages<TOOLS extends ToolSet = never>(
8585
case 'file': {
8686
content.push({
8787
type: 'file' as const,
88-
data: part.data,
88+
data: part.url,
8989
mediaType: part.mediaType ?? (part as any).mimeType, // TODO migration, remove
9090
});
9191
break;

‎packages/ai/core/types/ui-messages.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,9 +164,10 @@ export type FileUIPart = {
164164
mediaType: string;
165165

166166
/**
167-
* The base64 encoded data.
167+
* The URL of the file.
168+
* It can either be a URL to a hosted file or a [Data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs).
168169
*/
169-
data: string;
170+
url: string;
170171
};
171172

172173
/**

‎packages/ai/core/util/__snapshots__/process-chat-response.test.ts.snap

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -360,14 +360,14 @@ exports[`scenario: server provides file parts > should call the onFinish functio
360360
"type": "text",
361361
},
362362
{
363-
"data": "Hello World",
364363
"mediaType": "text/plain",
365364
"type": "file",
365+
"url": "data:text/plain;base64,SGVsbG8gV29ybGQ=",
366366
},
367367
{
368-
"data": "{"key": "value"}",
369368
"mediaType": "application/json",
370369
"type": "file",
370+
"url": "data:application/json;base64,eyJrZXkiOiJ2YWx1ZSJ9",
371371
},
372372
],
373373
"role": "assistant",
@@ -412,9 +412,9 @@ exports[`scenario: server provides file parts > should call the update function
412412
"type": "text",
413413
},
414414
{
415-
"data": "Hello World",
416415
"mediaType": "text/plain",
417416
"type": "file",
417+
"url": "data:text/plain;base64,SGVsbG8gV29ybGQ=",
418418
},
419419
],
420420
"revisionId": "id-2",
@@ -434,9 +434,9 @@ exports[`scenario: server provides file parts > should call the update function
434434
"type": "text",
435435
},
436436
{
437-
"data": "Hello World",
438437
"mediaType": "text/plain",
439438
"type": "file",
439+
"url": "data:text/plain;base64,SGVsbG8gV29ybGQ=",
440440
},
441441
],
442442
"revisionId": "id-3",
@@ -456,14 +456,14 @@ exports[`scenario: server provides file parts > should call the update function
456456
"type": "text",
457457
},
458458
{
459-
"data": "Hello World",
460459
"mediaType": "text/plain",
461460
"type": "file",
461+
"url": "data:text/plain;base64,SGVsbG8gV29ybGQ=",
462462
},
463463
{
464-
"data": "{"key": "value"}",
465464
"mediaType": "application/json",
466465
"type": "file",
466+
"url": "data:application/json;base64,eyJrZXkiOiJ2YWx1ZSJ9",
467467
},
468468
],
469469
"revisionId": "id-4",

‎packages/ai/core/util/data-stream-parts.test.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ describe('data-stream-parts', () => {
362362
describe('file stream part', () => {
363363
it('should format a file stream part', () => {
364364
const file = {
365-
data: 'file content',
365+
url: 'data:text/plain;base64,SGVsbG8gV29ybGQ=',
366366
mimeType: 'text/plain',
367367
};
368368

@@ -373,7 +373,7 @@ describe('data-stream-parts', () => {
373373

374374
it('should parse a file stream part', () => {
375375
const file = {
376-
data: 'file content',
376+
url: 'data:text/plain;base64,SGVsbG8gV29ybGQ=',
377377
mimeType: 'text/plain',
378378
};
379379

@@ -387,15 +387,15 @@ describe('data-stream-parts', () => {
387387
it('should throw an error if the file value is not an object', () => {
388388
const input = 'k:"not an object"';
389389
expect(() => parseDataStreamPart(input)).toThrow(
390-
'"file" parts expect an object with a "data" and "mimeType" property.',
390+
'"file" parts expect an object with a "url" and "mimeType" property.',
391391
);
392392
});
393393

394394
it('should throw an error if the file object is missing required properties', () => {
395395
const invalidFile = { name: 'test.txt' };
396396
const input = `k:${JSON.stringify(invalidFile)}`;
397397
expect(() => parseDataStreamPart(input)).toThrow(
398-
'"file" parts expect an object with a "data" and "mimeType" property.',
398+
'"file" parts expect an object with a "url" and "mimeType" property.',
399399
);
400400
});
401401
});

‎packages/ai/core/util/data-stream-parts.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -401,8 +401,8 @@ const fileStreamPart: DataStreamPart<
401401
'k',
402402
'file',
403403
{
404-
data: string; // base64 encoded data
405-
mimeType: string;
404+
url: string;
405+
mimeType: string; // TODO mediaType
406406
}
407407
> = {
408408
code: 'k',
@@ -411,16 +411,16 @@ const fileStreamPart: DataStreamPart<
411411
if (
412412
value == null ||
413413
typeof value !== 'object' ||
414-
!('data' in value) ||
415-
typeof value.data !== 'string' ||
414+
!('url' in value) ||
415+
typeof value.url !== 'string' ||
416416
!('mimeType' in value) ||
417417
typeof value.mimeType !== 'string'
418418
) {
419419
throw new Error(
420-
'"file" parts expect an object with a "data" and "mimeType" property.',
420+
'"file" parts expect an object with a "url" and "mimeType" property.',
421421
);
422422
}
423-
return { type: 'file', value: value as { data: string; mimeType: string } };
423+
return { type: 'file', value: value as { url: string; mimeType: string } };
424424
},
425425
};
426426

‎packages/ai/core/util/process-chat-response.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -712,12 +712,12 @@ describe('scenario: server provides file parts', () => {
712712
const stream = createDataProtocolStream([
713713
formatDataStreamPart('text', 'Here is a file:'),
714714
formatDataStreamPart('file', {
715-
data: 'Hello World',
715+
url: 'data:text/plain;base64,SGVsbG8gV29ybGQ=',
716716
mimeType: 'text/plain',
717717
}),
718718
formatDataStreamPart('text', 'And another one:'),
719719
formatDataStreamPart('file', {
720-
data: '{"key": "value"}',
720+
url: 'data:application/json;base64,eyJrZXkiOiJ2YWx1ZSJ9',
721721
mimeType: 'application/json',
722722
}),
723723
formatDataStreamPart('finish_step', {

‎packages/ai/core/util/process-chat-response.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ export async function processChatResponse({
170170
message.parts.push({
171171
type: 'file',
172172
mediaType: value.mimeType,
173-
data: value.data,
173+
url: value.url,
174174
});
175175

176176
execUpdate();

0 commit comments

Comments
 (0)