Skip to content

Commit edad14c

Browse files
PatrickFarleyPatrickFarley
committed
change schema
1 parent 1f632e5 commit edad14c

File tree

3 files changed

+104
-44
lines changed

3 files changed

+104
-44
lines changed

‎articles/ai-services/openai/how-to/gpt-with-vision.md

Lines changed: 73 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,15 @@ The following is a sample request body. The format is the same as the chat compl
4747
},
4848
{
4949
"role": "user",
50-
"content": [
51-
{ "type": "text", "text": "Describe this picture:" },
52-
{ "type": "image_url", "url": "<URL or base-64-encoded image>" }
50+
"content": [
51+
{
52+
"type": "text",
53+
"text": "Describe this picture:"
54+
},
55+
{
56+
"type": "image_url",
57+
"image_url": "<URL or base-64-encoded image>"
58+
}
5359
]
5460
}
5561
],
@@ -64,31 +70,68 @@ The API response should look like the following.
6470

6571
```json
6672
{
67-
"id": "chatcmpl-8Uyxu7xpvngMZnvMhVAPpiI3laUef",
73+
"id": "chatcmpl-8VAVx58veW9RCm5K1ttmxU6Cm4XDX",
6874
"object": "chat.completion",
69-
"created": 1702394882,
75+
"created": 1702439277,
7076
"model": "gpt-4",
71-
"choices":
72-
[
77+
"prompt_filter_results": [
7378
{
74-
"finish_details":
75-
{
79+
"prompt_index": 0,
80+
"content_filter_results": {
81+
"hate": {
82+
"filtered": false,
83+
"severity": "safe"
84+
},
85+
"self_harm": {
86+
"filtered": false,
87+
"severity": "safe"
88+
},
89+
"sexual": {
90+
"filtered": false,
91+
"severity": "safe"
92+
},
93+
"violence": {
94+
"filtered": false,
95+
"severity": "safe"
96+
}
97+
}
98+
}
99+
],
100+
"choices": [
101+
{
102+
"finish_details": {
76103
"type": "stop",
77104
"stop": "<|fim_suffix|>"
78105
},
79106
"index": 0,
80-
"message":
81-
{
107+
"message": {
82108
"role": "assistant",
83-
"content": "This picture depicts a grayscale image of an individual showcasing dark hair combed to one side, and the tips of their ears are also visible. The person appears to be dressed in a casual outfit with a hint of a garment that has a collar or neckline. The background is a plain and neutral shade, creating a stark contrast with the subject in the foreground."
109+
"content": "The picture shows an individual dressed in formal attire, which includes a black tuxedo with a black bow tie. There is an American flag on the left lapel of the individual's jacket. The background is predominantly blue with white text that reads \"THE KENNEDY PROFILE IN COURAGE AWARD\" and there are also visible elements of the flag of the United States placed behind the individual."
110+
},
111+
"content_filter_results": {
112+
"hate": {
113+
"filtered": false,
114+
"severity": "safe"
115+
},
116+
"self_harm": {
117+
"filtered": false,
118+
"severity": "safe"
119+
},
120+
"sexual": {
121+
"filtered": false,
122+
"severity": "safe"
123+
},
124+
"violence": {
125+
"filtered": false,
126+
"severity": "safe"
127+
}
84128
}
85129
}
86130
],
87-
"usage":
88-
{
89-
"prompt_tokens": 816,
90-
"completion_tokens": 71,
91-
"total_tokens": 887
131+
"usage": {
132+
"prompt_tokens": 1156,
133+
"completion_tokens": 80,
134+
"total_tokens": 1236
92135
}
93136
}
94137
```
@@ -147,16 +190,22 @@ You must also include the `enhancements` and `dataSources` objects. `enhancement
147190
"key": "<your_computer_vision_key>"
148191
}
149192
}],
150-
"messages": [
193+
"messages": [
151194
{
152-
"role": "system",
153-
"content": "You are a helpful assistant."
195+
"role": "system",
196+
"content": "You are a helpful assistant."
154197
},
155198
{
156-
"role": "user",
157-
"content": [
158-
{ "type": "text", "text": "Describe this picture:" },
159-
{ "type": "image_url", "url": "<URL or base-64-encoded image>" }
199+
"role": "user",
200+
"content": [
201+
{
202+
"type": "text",
203+
"text": "Describe this picture:"
204+
},
205+
{
206+
"type": "image_url",
207+
"image_url": "<URL or base-64-encoded image>"
208+
}
160209
]
161210
}
162211
],

‎articles/ai-services/openai/includes/gpt-v-rest.md

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -62,19 +62,25 @@ Create a new Python file named _quickstart.py_. Open the new file in your prefer
6262
data = {
6363
"messages": [
6464
{ "role": "system", "content": "You are a helpful assistant." },
65-
{ "role": "user", "content": [
66-
{ "type": "text", "text": "Describe this picture:" },
67-
{ "type": "image_url", "url": "<URL or base-64-encoded image>" }
68-
] }
65+
{ "role": "user", "content": [
66+
{
67+
"type": "text",
68+
"text": "Describe this picture:"
69+
},
70+
{
71+
"type": "image_url",
72+
"image_url": "<URL or base 64 encoded image>" # Can url or data url with base64 image string
73+
}
74+
] }
6975
],
70-
"max_tokens": 100
76+
"max_tokens": 2000
7177
}
7278

7379
# Make the API call
7480
response = requests.post(endpoint, headers=headers, data=json.dumps(data))
7581

7682
print(f"Status Code: {response.status_code}")
77-
print(response.text)
83+
print(response.text)
7884
```
7985

8086
1. Make the following changes:
@@ -130,26 +136,32 @@ The **object grounding** integration brings a new layer to data analysis and use
130136
"dataSources": [
131137
{
132138
"type": "AzureComputerVision",
133-
"parameters": {
134139
"endpoint": " <your_computer_vision_endpoint> ",
135-
"key": "<your_computer_vision_key>"
136-
}
140+
"key": "<your_computer_vision_key>",
141+
"indexName": "test-products"
137142
}],
138143
"messages": [
139-
{ "role": "system", "content": "You are a helpful assistant." },
140-
{ "role": "user", "content": [
141-
{ "type": "text", "text": "Describe this picture:" },
142-
{ "type": "image_url", "url": "<URL or base-64-encoded image>" }
143-
]}
144+
{ "role": "system", "content": "You are a helpful assistant." },
145+
{ "role": "user",
146+
"content": [
147+
{
148+
"type": "text",
149+
"text": "Describe this picture:"
150+
},
151+
{
152+
"type": "image_url",
153+
"image_url": "<URL or base 64 encoded image>" # Can be url or data url with base64 image string
154+
}
155+
]}
144156
],
145-
"max_tokens": 100
157+
"max_tokens": 2000
146158
}
147159

148160
# Make the API call
149161
response = requests.post(endpoint, headers=headers, data=json.dumps(data))
150-
162+
151163
print(f"Status Code: {response.status_code}")
152-
print(response.text)
164+
print(response.text)
153165
```
154166

155167
1. Make the following changes:

‎articles/ai-services/openai/reference.md

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -230,23 +230,22 @@ curl https://YOUR_RESOURCE_NAME.openai.azure.com/openai/deployments/YOUR_DEPLOYM
230230
-H "Content-Type: application/json" \
231231
-H "api-key: YOUR_API_KEY" \
232232
-d '{"messages":[{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Does Azure OpenAI support customer managed keys?"},{"role": "assistant", "content": "Yes, customer managed keys are supported by Azure OpenAI."},{"role": "user", "content": "Do other Azure AI services support this too?"}]}'
233-
234233
```
235234

236235
**Chat with vision**
237236
```console
238237
curl https://YOUR_RESOURCE_NAME.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT_NAME/chat/completions?api-version=2023-12-01-preview \
239238
-H "Content-Type: application/json" \
240239
-H "api-key: YOUR_API_KEY" \
241-
-d '{"messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":[{"type":"text","text":"Describe this picture:"},{"type":"image_url","url":"https://learn.microsoft.com/azure/ai-services/computer-vision/media/quickstarts/presentation.png"}]}]}'
240+
-d '{"messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":[{"type":"text","text":"Describe this picture:"},{ "type": "image_url", "image_url": { "url": https://learn.microsoft.com/azure/ai-services/computer-vision/media/quickstarts/presentation.png, "detail": "high" } }]}]}'
242241
```
243242

244243
**Enhanced chat with vision**
245244
```console
246245
curl https://YOUR_RESOURCE_NAME.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT_NAME/extensions/chat/completions?api-version=2023-12-01-preview \
247246
-H "Content-Type: application/json" \
248247
-H "api-key: YOUR_API_KEY" \
249-
-d '{"enhancements":{"ocr":{"enabled":true},"grounding":{"enabled":true}},"dataSources":[{"type":"AzureComputerVision","parameters":{"endpoint":" <Computer Vision Resource Endpoint> ","key":"<Computer Vision Resource Key>"}}],"messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":[{"type":"text","text":"Describe this picture:"},{"type":"image_url","url":"https://learn.microsoft.com/azure/ai-services/computer-vision/media/quickstarts/presentation.png"}]}]}'
248+
-d '{"enhancements":{"ocr":{"enabled":true},"grounding":{"enabled":true}},"dataSources":[{"type":"AzureComputerVision","parameters":{"endpoint":" <Computer Vision Resource Endpoint> ","key":"<Computer Vision Resource Key>"}}],"messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":[{"type":"text","text":"Describe this picture:"},{"type":"image_url","image_url":"https://learn.microsoft.com/azure/ai-services/computer-vision/media/quickstarts/presentation.png"}]}]}'
250249
```
251250

252251
#### Example response

0 commit comments

Comments
 (0)