Skip to content

Commit f04fb4a

Browse files
lgrammellgrammel
authored
chore (ai): replace useChat attachments with file ui parts (#6071)
## Background With the switch to UI message parts, attachments can be replaced with FileUIParts to enable using the same code for assistant and user file parts. ## Summary Replace attachments with file ui parts.
1 parent c34ccd7 commit f04fb4a

26 files changed

+1202
-1206
lines changed

‎.changeset/lemon-actors-invite.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'ai': major
3+
---
4+
5+
chore (ai): replace useChat attachments with file ui parts

‎examples/next-fastapi/app/(examples)/03-chat-attachments/page.tsx

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import { Card } from '@/app/components';
44
/* eslint-disable @next/next/no-img-element */
5-
import { getTextFromDataUrl } from 'ai';
65
import { useChat } from '@ai-sdk/react';
76
import { useRef, useState } from 'react';
87

@@ -20,26 +19,22 @@ export default function Page() {
2019
{messages.map(message => (
2120
<div key={message.id} className="flex flex-row gap-2">
2221
<div className="flex-shrink-0 w-24 text-zinc-500">{`${message.role}: `}</div>
23-
2422
<div className="flex flex-col gap-2">
25-
{message.content}
26-
27-
<div className="flex flex-row gap-2">
28-
{message.experimental_attachments?.map((attachment, index) =>
29-
attachment.contentType?.includes('image/') ? (
30-
<img
31-
key={`${message.id}-${index}`}
32-
className="w-24 rounded-md"
33-
src={attachment.url}
34-
alt={attachment.name}
35-
/>
36-
) : attachment.contentType?.includes('text/') ? (
37-
<div className="w-32 h-24 p-2 overflow-hidden text-xs border rounded-md ellipsis text-zinc-500">
38-
{getTextFromDataUrl(attachment.url)}
23+
{message.parts.map((part, index) => {
24+
if (part.type === 'text') {
25+
return <div key={index}>{part.text}</div>;
26+
}
27+
if (
28+
part.type === 'file' &&
29+
part.mediaType?.startsWith('image/')
30+
) {
31+
return (
32+
<div key={index}>
33+
<img className="rounded-md w-60" src={part.url} />
3934
</div>
40-
) : null,
41-
)}
42-
</div>
35+
);
36+
}
37+
})}
4338
</div>
4439
</div>
4540
))}
@@ -49,9 +44,7 @@ export default function Page() {
4944

5045
<form
5146
onSubmit={event => {
52-
handleSubmit(event, {
53-
experimental_attachments: files,
54-
});
47+
handleSubmit(event, { files });
5548
setFiles(undefined);
5649

5750
if (fileInputRef.current) {

‎examples/next-openai/app/use-chat-attachments-append/page.tsx

Lines changed: 24 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
'use client';
22

33
/* eslint-disable @next/next/no-img-element */
4-
import { getTextFromDataUrl } from 'ai';
54
import { useChat } from '@ai-sdk/react';
5+
import { convertFileListToFileUIParts } from 'ai';
66
import { useRef, useState } from 'react';
77

88
export default function Page() {
@@ -21,43 +21,38 @@ export default function Page() {
2121
<div className="flex-shrink-0 w-24 text-zinc-500">{`${message.role}: `}</div>
2222

2323
<div className="flex flex-col gap-2">
24-
{message.content}
25-
26-
<div className="flex flex-row gap-2">
27-
{message.experimental_attachments?.map((attachment, index) =>
28-
attachment.contentType?.includes('image/') ? (
29-
<img
30-
key={`${message.id}-${index}`}
31-
className="w-24 rounded-md"
32-
src={attachment.url}
33-
alt={attachment.name}
34-
/>
35-
) : attachment.contentType?.includes('text/') ? (
36-
<div className="w-32 h-24 p-2 overflow-hidden text-xs border rounded-md ellipsis text-zinc-500">
37-
{getTextFromDataUrl(attachment.url)}
24+
{message.parts.map((part, index) => {
25+
if (part.type === 'text') {
26+
return <div key={index}>{part.text}</div>;
27+
}
28+
if (
29+
part.type === 'file' &&
30+
part.mediaType?.startsWith('image/')
31+
) {
32+
return (
33+
<div key={index}>
34+
<img className="rounded-md w-60" src={part.url} />
3835
</div>
39-
) : null,
40-
)}
41-
</div>
36+
);
37+
}
38+
})}
4239
</div>
4340
</div>
4441
))}
4542
</div>
4643

4744
<form
48-
onSubmit={event => {
45+
onSubmit={async event => {
4946
event.preventDefault();
5047

51-
append(
52-
{
53-
role: 'user',
54-
content: input,
55-
parts: [{ type: 'text', text: input }],
56-
},
57-
{
58-
experimental_attachments: files,
59-
},
60-
);
48+
append({
49+
role: 'user',
50+
content: input,
51+
parts: [
52+
...(await convertFileListToFileUIParts(files)),
53+
{ type: 'text', text: input },
54+
],
55+
});
6156

6257
setFiles(undefined);
6358
setInput('');

‎examples/next-openai/app/use-chat-attachments-url/page.tsx

Lines changed: 29 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@
44
import { useChat } from '@ai-sdk/react';
55
import { useRef, useState } from 'react';
66
import { upload } from '@vercel/blob/client';
7-
import { Attachment } from 'ai';
7+
import { FileUIPart } from 'ai';
88

99
export default function Page() {
1010
const { messages, input, handleSubmit, handleInputChange, status } = useChat({
1111
api: '/api/chat',
1212
});
1313

14-
const [attachments, setAttachments] = useState<Attachment[]>([]);
14+
const [files, setFiles] = useState<FileUIPart[]>([]);
1515
const [isUploading, setIsUploading] = useState<boolean>(false);
1616
const fileInputRef = useRef<HTMLInputElement>(null);
1717

@@ -22,20 +22,17 @@ export default function Page() {
2222
<div key={message.id} className="flex flex-row gap-2">
2323
<div className="flex-shrink-0 w-24 text-zinc-500">{`${message.role}: `}</div>
2424

25-
<div className="flex flex-col gap-2">
26-
{message.content}
27-
28-
<div className="flex flex-row gap-2">
29-
{message.experimental_attachments?.map((attachment, index) => (
30-
<img
31-
key={`${message.id}-${index}`}
32-
className="w-24 rounded-md"
33-
src={attachment.url}
34-
alt={attachment.name}
35-
/>
36-
))}
37-
</div>
38-
</div>
25+
{message.parts.map((part, index) => {
26+
if (part.type === 'text') {
27+
return <div key={index}>{part.text}</div>;
28+
}
29+
if (
30+
part.type === 'file' &&
31+
part.mediaType?.startsWith('image/')
32+
) {
33+
return <img key={index} src={part.url} />;
34+
}
35+
})}
3936
</div>
4037
))}
4138
</div>
@@ -47,11 +44,9 @@ export default function Page() {
4744
return;
4845
}
4946

50-
handleSubmit(event, {
51-
experimental_attachments: attachments,
52-
});
47+
handleSubmit(event, { files });
5348

54-
setAttachments([]);
49+
setFiles([]);
5550

5651
if (fileInputRef.current) {
5752
fileInputRef.current.value = '';
@@ -60,16 +55,18 @@ export default function Page() {
6055
className="fixed bottom-0 flex flex-col w-full gap-2 p-2"
6156
>
6257
<div className="fixed flex flex-row items-end gap-2 right-2 bottom-14">
63-
{Array.from(attachments)
64-
.filter(attachment => attachment.contentType?.startsWith('image/'))
65-
.map(attachment => (
66-
<div key={attachment.name}>
58+
{Array.from(files)
59+
.filter(file => file.mediaType?.startsWith('image/'))
60+
.map(file => (
61+
<div key={file.url}>
6762
<img
6863
className="w-24 rounded-md"
69-
src={attachment.url}
70-
alt={attachment.name}
64+
src={file.url}
65+
// alt={file.name} TODO filename support
7166
/>
72-
<span className="text-sm text-zinc-500">{attachment.name}</span>
67+
<span className="text-sm text-zinc-500">
68+
{'name' /* TODO file.name*/}
69+
</span>
7370
</div>
7471
))}
7572
</div>
@@ -85,11 +82,12 @@ export default function Page() {
8582
handleUploadUrl: '/api/files',
8683
});
8784

88-
setAttachments(prevAttachments => [
89-
...prevAttachments,
85+
setFiles(prevFiles => [
86+
...prevFiles,
9087
{
91-
name: file.name,
92-
contentType: blob.contentType,
88+
type: 'file' as const,
89+
// name: file.name, TODO filename support
90+
mediaType: blob.contentType ?? '*/*',
9391
url: blob.url,
9492
},
9593
]);

‎examples/next-openai/app/use-chat-attachments/page.tsx

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
'use client';
22

33
/* eslint-disable @next/next/no-img-element */
4-
import { getTextFromDataUrl } from 'ai';
54
import { useChat } from '@ai-sdk/react';
65
import { useRef, useState } from 'react';
76

@@ -21,34 +20,29 @@ export default function Page() {
2120
<div className="flex-shrink-0 w-24 text-zinc-500">{`${message.role}: `}</div>
2221

2322
<div className="flex flex-col gap-2">
24-
{message.content}
25-
26-
<div className="flex flex-row gap-2">
27-
{message.experimental_attachments?.map((attachment, index) =>
28-
attachment.contentType?.includes('image/') ? (
29-
<img
30-
key={`${message.id}-${index}`}
31-
className="w-24 rounded-md"
32-
src={attachment.url}
33-
alt={attachment.name}
34-
/>
35-
) : attachment.contentType?.includes('text/') ? (
36-
<div className="w-32 h-24 p-2 overflow-hidden text-xs border rounded-md ellipsis text-zinc-500">
37-
{getTextFromDataUrl(attachment.url)}
23+
{message.parts.map((part, index) => {
24+
if (part.type === 'text') {
25+
return <div key={index}>{part.text}</div>;
26+
}
27+
if (
28+
part.type === 'file' &&
29+
part.mediaType?.startsWith('image/')
30+
) {
31+
return (
32+
<div key={index}>
33+
<img className="rounded-md w-60" src={part.url} />
3834
</div>
39-
) : null,
40-
)}
41-
</div>
35+
);
36+
}
37+
})}
4238
</div>
4339
</div>
4440
))}
4541
</div>
4642

4743
<form
4844
onSubmit={event => {
49-
handleSubmit(event, {
50-
experimental_attachments: files,
51-
});
45+
handleSubmit(event, { files });
5246
setFiles(undefined);
5347

5448
if (fileInputRef.current) {

‎packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap

Lines changed: 0 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -413,45 +413,3 @@ exports[`convertToCoreMessages > multiple messages > should handle conversation
413413
},
414414
]
415415
`;
416-
417-
exports[`convertToCoreMessages > user message > should handle user message with attachment URLs (file) 1`] = `
418-
[
419-
{
420-
"content": [
421-
{
422-
"text": "Check this document",
423-
"type": "text",
424-
},
425-
{
426-
"data": "dGVzdA==",
427-
"mediaType": "application/pdf",
428-
"type": "file",
429-
},
430-
],
431-
"role": "user",
432-
},
433-
]
434-
`;
435-
436-
exports[`convertToCoreMessages > user message > should handle user message with attachment URLs 1`] = `
437-
[
438-
{
439-
"content": [
440-
{
441-
"text": "Check this image",
442-
"type": "text",
443-
},
444-
{
445-
"image": Uint8Array [
446-
116,
447-
101,
448-
115,
449-
116,
450-
],
451-
"type": "image",
452-
},
453-
],
454-
"role": "user",
455-
},
456-
]
457-
`;

0 commit comments

Comments
 (0)