Skip to content

Commit b2a742e

Browse files
aabmasslmolkova
aabmass
and
lmolkova
authored
Add multimodal uri, file, and blob parts to GenAI JSON Schemas (#2754)
Co-authored-by: Liudmila Molkova <[email protected]>
1 parent 7df6f86 commit b2a742e

File tree

6 files changed

+654
-10
lines changed

6 files changed

+654
-10
lines changed

‎.chloggen/genai-multimodal-1556.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
change_type: enhancement
2+
component: gen_ai
3+
note: Add multimodal `uri`, `file`, and `blob` parts to GenAI JSON Schemas
4+
issues: [1556]

‎docs/gen-ai/gen-ai-input-messages.json‎

Lines changed: 166 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,54 @@
11
{
22
"$defs": {
3+
"BlobPart": {
4+
"description": "Represents blob binary data sent inline to the model",
5+
"properties": {
6+
"type": {
7+
"const": "blob",
8+
"description": "The type of the content captured in this part.",
9+
"title": "Type",
10+
"type": "string"
11+
},
12+
"mime_type": {
13+
"anyOf": [
14+
{
15+
"type": "string"
16+
},
17+
{
18+
"type": "null"
19+
}
20+
],
21+
"default": null,
22+
"description": "The IANA MIME type of the attached data.",
23+
"title": "Mime Type"
24+
},
25+
"modality": {
26+
"anyOf": [
27+
{
28+
"$ref": "#/$defs/Modality"
29+
},
30+
{
31+
"type": "string"
32+
}
33+
],
34+
"description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.",
35+
"title": "Modality"
36+
},
37+
"content": {
38+
"description": "Raw bytes of the attached data. This field SHOULD be encoded as a base64 string when transmitted as JSON.",
39+
"format": "binary",
40+
"title": "Content",
41+
"type": "string"
42+
}
43+
},
44+
"required": [
45+
"type",
46+
"modality",
47+
"content"
48+
],
49+
"title": "BlobPart",
50+
"type": "object"
51+
},
352
"ChatMessage": {
453
"additionalProperties": true,
554
"properties": {
@@ -28,6 +77,15 @@
2877
{
2978
"$ref": "#/$defs/ToolCallResponsePart"
3079
},
80+
{
81+
"$ref": "#/$defs/BlobPart"
82+
},
83+
{
84+
"$ref": "#/$defs/FilePart"
85+
},
86+
{
87+
"$ref": "#/$defs/UriPart"
88+
},
3189
{
3290
"$ref": "#/$defs/ReasoningPart"
3391
},
@@ -60,6 +118,55 @@
60118
"title": "ChatMessage",
61119
"type": "object"
62120
},
121+
"FilePart": {
122+
"additionalProperties": true,
123+
"description": "Represents an external referenced file sent to the model by file id",
124+
"properties": {
125+
"type": {
126+
"const": "file",
127+
"description": "The type of the content captured in this part.",
128+
"title": "Type",
129+
"type": "string"
130+
},
131+
"mime_type": {
132+
"anyOf": [
133+
{
134+
"type": "string"
135+
},
136+
{
137+
"type": "null"
138+
}
139+
],
140+
"default": null,
141+
"description": "The IANA MIME type of the attached data.",
142+
"title": "Mime Type"
143+
},
144+
"modality": {
145+
"anyOf": [
146+
{
147+
"$ref": "#/$defs/Modality"
148+
},
149+
{
150+
"type": "string"
151+
}
152+
],
153+
"description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.",
154+
"title": "Modality"
155+
},
156+
"file_id": {
157+
"description": "An identifier referencing a file that was pre-uploaded to the provider.",
158+
"title": "File Id",
159+
"type": "string"
160+
}
161+
},
162+
"required": [
163+
"type",
164+
"modality",
165+
"file_id"
166+
],
167+
"title": "FilePart",
168+
"type": "object"
169+
},
63170
"GenericPart": {
64171
"additionalProperties": true,
65172
"description": "Represents an arbitrary message part with any type and properties.\nThis allows for extensibility with custom message part types.",
@@ -76,6 +183,15 @@
76183
"title": "GenericPart",
77184
"type": "object"
78185
},
186+
"Modality": {
187+
"enum": [
188+
"image",
189+
"video",
190+
"audio"
191+
],
192+
"title": "Modality",
193+
"type": "string"
194+
},
79195
"ReasoningPart": {
80196
"additionalProperties": true,
81197
"description": "Represents reasoning/thinking content received from the model.",
@@ -207,6 +323,55 @@
207323
],
208324
"title": "ToolCallResponsePart",
209325
"type": "object"
326+
},
327+
"UriPart": {
328+
"additionalProperties": true,
329+
"description": "Represents an external referenced file sent to the model by URI",
330+
"properties": {
331+
"type": {
332+
"const": "uri",
333+
"description": "The type of the content captured in this part.",
334+
"title": "Type",
335+
"type": "string"
336+
},
337+
"mime_type": {
338+
"anyOf": [
339+
{
340+
"type": "string"
341+
},
342+
{
343+
"type": "null"
344+
}
345+
],
346+
"default": null,
347+
"description": "The IANA MIME type of the attached data.",
348+
"title": "Mime Type"
349+
},
350+
"modality": {
351+
"anyOf": [
352+
{
353+
"$ref": "#/$defs/Modality"
354+
},
355+
{
356+
"type": "string"
357+
}
358+
],
359+
"description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.",
360+
"title": "Modality"
361+
},
362+
"uri": {
363+
"description": "A URI referencing attached data. It should not be a base64 data URL, which should use the `blob` part instead. The URI may use a scheme known to the provider api (e.g. `gs://bucket/object.png`), or be a publicly accessible location.",
364+
"title": "Uri",
365+
"type": "string"
366+
}
367+
},
368+
"required": [
369+
"type",
370+
"modality",
371+
"uri"
372+
],
373+
"title": "UriPart",
374+
"type": "object"
210375
}
211376
},
212377
"description": "Represents the list of input messages sent to the model.",
@@ -215,4 +380,4 @@
215380
},
216381
"title": "InputMessages",
217382
"type": "array"
218-
}
383+
}

0 commit comments

Comments
 (0)