Skip to content

Commit c893809

Browse files
Yun-KimYun-Kim
authored
fix(openai): correctly tag image inputs for chat completions [backport #7759 to 2.3] (#7814)
This backports #7759 to 2.3. This PR adds a step to stringify input messages before tagging in the OpenAI chat completions endpoint. Previously, we had assumed that messages.content would always be a string (which was true until OpenAI recently added the image input feature to the chat completions endpoint), but it can now be an array of str-str dictionaries. ## Testing Strategy Regression tests have been added, and manual testing has also confirmed that the error reported on #7737 does not appear. ## Checklist - [x] Change(s) are motivated and described in the PR description. - [x] Testing strategy is described if automated tests are not included in the PR. - [x] Risk is outlined (performance impact, potential for breakage, maintainability, etc). - [x] Change is maintainable (easy to change, telemetry, documentation). - [x] [Library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) are followed. If no release note is required, add label `changelog/no-changelog`. - [x] Documentation is included (in-code, generated user docs, [public corp docs](https://github.com/DataDog/documentation/)). - [x] Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Title is accurate. - [x] No unnecessary changes are introduced. - [x] Description motivates each change. - [x] Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes unless absolutely necessary. - [x] Testing strategy adequately addresses listed risk(s). - [x] Change is maintainable (easy to change, telemetry, documentation). - [x] Release note makes sense to a user of the library. - [x] Reviewer has explicitly acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment. - [x] Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) - [x] If this PR touches code that signs or publishes builds or packages, or handles credentials of any kind, I've requested a review from `@DataDog/security-design-and-guidance`. - [x] This PR doesn't touch any of that.
1 parent 0bc6537 commit c893809

File tree

7 files changed

+295
-2
lines changed

7 files changed

+295
-2
lines changed

‎ddtrace/contrib/openai/_endpoint_hooks.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ def _record_request(self, pin, integration, span, args, kwargs):
256256
messages = kwargs.get("messages", [])
257257
if integration.is_pc_sampled_span(span):
258258
for idx, m in enumerate(messages):
259-
content = integration.trunc(m.get("content", ""))
259+
content = integration.trunc(str(m.get("content", "")))
260260
role = integration.trunc(m.get("role", ""))
261261
name = integration.trunc(m.get("name", ""))
262262
span.set_tag_str("openai.request.messages.%d.content" % idx, content)
@@ -284,7 +284,7 @@ def _record_response(self, pin, integration, span, args, kwargs, resp, error):
284284
span.set_tag_str("openai.response.model", resp.model or "")
285285
for choice in choices:
286286
idx = choice.index
287-
finish_reason = choice.finish_reason
287+
finish_reason = getattr(choice, "finish_reason", None)
288288
message = choice.message
289289
if finish_reason is not None:
290290
span.set_tag_str("openai.response.choices.%d.finish_reason" % idx, str(finish_reason))

‎releasenotes/notes/fix-openai-chat-image-input-d664b6ef30492e2c.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
fixes:
3+
- |
4+
openai: This fix resolves an issue where tagging image inputs in the chat completions endpoint resulted in attribute errors.

‎tests/contrib/openai/cassettes/v0/chat_completion_image_input.yaml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
interactions:
2+
- request:
3+
body: '{"model": "gpt-4-vision-preview", "messages": [{"role": "user", "content":
4+
[{"type": "text", "text": "What\u2019s in this image?"}, {"type": "image_url",
5+
"image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}]}]}'
6+
headers:
7+
Accept:
8+
- '*/*'
9+
Accept-Encoding:
10+
- gzip, deflate
11+
Connection:
12+
- keep-alive
13+
Content-Length:
14+
- '332'
15+
Content-Type:
16+
- application/json
17+
User-Agent:
18+
- OpenAI/v1 PythonBindings/0.27.2
19+
X-OpenAI-Client-User-Agent:
20+
- '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version":
21+
"3.11.1", "platform": "macOS-14.1.1-arm64-arm-64bit", "publisher": "openai",
22+
"uname": "Darwin 23.1.0 Darwin Kernel Version 23.1.0: Mon Oct 9 21:27:24
23+
PDT 2023; root:xnu-10002.41.9~6/RELEASE_ARM64_T6000 arm64 arm"}'
24+
method: POST
25+
uri: https://api.openai.com/v1/chat/completions
26+
response:
27+
body:
28+
string: !!binary |
29+
H4sIAAAAAAAAA0yPQWvDMAyF/4rweRkJLW3pfbuMwWCDjY1R1FiJvdiWsZWmpfS/DwdaehPSe5/e
30+
Oyur1RZUa1BaH121efteP73arqZaf+HmdHoe34euHvzn9BLVAyje/1ErV89jyz46EsuhHNtEKFSI
31+
zbpumnqxWa8eQHnW5Iqlj1Itq6apV9XBZsuhiokOlqZiHjP2pLZwVjGxj7ITHijkAmuaTaHfft2d
32+
Cl9Y0N3LF8tL0Ru2LZXNz1l5yjd+YlcGhTnbLBhkzs5BKMzNPgyB9dgTZMNTBoRMiQJBQBkTOnAY
33+
dG4xErQcCsOGHrgDhIlZU4A9Y9ITugHEoAAdhYLOICbx2BtV0nU22Gx2mgSty3MuOcU5l8fjtU1R
34+
2qDpqLZQX34v/wAAAP//AwBzg8RBsQEAAA==
35+
headers:
36+
CF-Cache-Status:
37+
- DYNAMIC
38+
CF-RAY:
39+
- 82cbdfefb813436e-EWR
40+
Connection:
41+
- keep-alive
42+
Content-Encoding:
43+
- gzip
44+
Content-Type:
45+
- application/json
46+
Date:
47+
- Mon, 27 Nov 2023 16:51:18 GMT
48+
Server:
49+
- cloudflare
50+
Set-Cookie:
51+
- __cf_bm=d6ov6YwrqemojSGt8iO9_0Qf3UJhwlgQMcoKdnoRbwg-1701103878-0-ASEB1DDqa/JLbWcuxqNOaYCngY4tGk9Q8m3aVpU4jAjGua7OpatkiSrzIPI9rRMHQUqGKdiJYa52zH3JS1iq5Wk=;
52+
path=/; expires=Mon, 27-Nov-23 17:21:18 GMT; domain=.api.openai.com; HttpOnly;
53+
Secure; SameSite=None
54+
- _cfuvid=ALaEbe03egqFuF8IP_iPjk4QMdgVjcdkH1QEiSb2ric-1701103878609-0-604800000;
55+
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
56+
Transfer-Encoding:
57+
- chunked
58+
alt-svc:
59+
- h3=":443"; ma=86400
60+
openai-model:
61+
- gpt-4-1106-vision-preview
62+
openai-organization:
63+
- datadog-4
64+
openai-processing-ms:
65+
- '4003'
66+
openai-version:
67+
- '2020-10-01'
68+
strict-transport-security:
69+
- max-age=15724800; includeSubDomains
70+
x-ratelimit-limit-requests:
71+
- '100'
72+
x-ratelimit-limit-tokens:
73+
- '150000'
74+
x-ratelimit-remaining-requests:
75+
- '98'
76+
x-ratelimit-remaining-tokens:
77+
- '149977'
78+
x-ratelimit-reset-requests:
79+
- 25m47.91s
80+
x-ratelimit-reset-tokens:
81+
- 9ms
82+
x-request-id:
83+
- fed91798d778620c33cc96e1c49f0300
84+
status:
85+
code: 200
86+
message: OK
87+
version: 1

‎tests/contrib/openai/cassettes/v1/chat_completion_image_input.yaml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
interactions:
2+
- request:
3+
body: '{"messages": [{"role": "user", "content": [{"type": "text", "text": "What\u2019s
4+
in this image?"}, {"type": "image_url", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}]}],
5+
"model": "gpt-4-vision-preview"}'
6+
headers:
7+
accept:
8+
- application/json
9+
accept-encoding:
10+
- gzip, deflate
11+
connection:
12+
- keep-alive
13+
content-length:
14+
- '332'
15+
content-type:
16+
- application/json
17+
host:
18+
- api.openai.com
19+
user-agent:
20+
- OpenAI/Python 1.1.1
21+
x-stainless-arch:
22+
- arm64
23+
x-stainless-lang:
24+
- python
25+
x-stainless-os:
26+
- MacOS
27+
x-stainless-package-version:
28+
- 1.1.1
29+
x-stainless-runtime:
30+
- CPython
31+
x-stainless-runtime-version:
32+
- 3.11.1
33+
method: POST
34+
uri: https://api.openai.com/v1/chat/completions
35+
response:
36+
content: '{"id": "chatcmpl-8PZ7EMif0e0dXa8yyFuSkf0kmWwKp", "object": "chat.completion",
37+
"created": 1701103876, "model": "gpt-4-1106-vision-preview", "usage": {"prompt_tokens":
38+
1118, "completion_tokens": 16, "total_tokens": 1134}, "choices": [{"message":
39+
{"role": "assistant", "content": "The image shows a serene natural landscape consisting of a wooden boardwalk that extends through"}, "finish_details": {"type": "max_tokens"},
40+
"index": 0}]}'
41+
headers:
42+
CF-Cache-Status:
43+
- DYNAMIC
44+
CF-RAY:
45+
- 82cbdb89482dc420-EWR
46+
Connection:
47+
- keep-alive
48+
Content-Encoding:
49+
- gzip
50+
Content-Type:
51+
- application/json
52+
Date:
53+
- Mon, 27 Nov 2023 16:48:19 GMT
54+
Server:
55+
- cloudflare
56+
Set-Cookie:
57+
- __cf_bm=GG9trWAPd86aqXS0Gt6guzjR2FLum2Fa5zkfYbUCNUw-1701103699-0-Aetjz9Ne380ZTQMFwc/pySteL/hStTE57jbQX8ddzByy+YCO3xU2xryQQu/rV/IAJrmhMOsMpdonkyX6+/lGwdY=;
58+
path=/; expires=Mon, 27-Nov-23 17:18:19 GMT; domain=.api.openai.com; HttpOnly;
59+
Secure; SameSite=None
60+
- _cfuvid=1aDtGGCysQtwRa_0YB6rXUoELGu20x4HAE_1R_jGQU8-1701103699770-0-604800000;
61+
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
62+
Transfer-Encoding:
63+
- chunked
64+
alt-svc:
65+
- h3=":443"; ma=86400
66+
openai-model:
67+
- gpt-4-1106-vision-preview
68+
openai-organization:
69+
- datadog-4
70+
openai-processing-ms:
71+
- '5254'
72+
openai-version:
73+
- '2020-10-01'
74+
strict-transport-security:
75+
- max-age=15724800; includeSubDomains
76+
x-ratelimit-limit-requests:
77+
- '100'
78+
x-ratelimit-limit-tokens:
79+
- '150000'
80+
x-ratelimit-remaining-requests:
81+
- '98'
82+
x-ratelimit-remaining-tokens:
83+
- '149977'
84+
x-ratelimit-reset-requests:
85+
- 14m24s
86+
x-ratelimit-reset-tokens:
87+
- 9ms
88+
x-request-id:
89+
- a2dab4f27488b3acab1cfe854feb6895
90+
http_version: HTTP/1.1
91+
status_code: 200
92+
version: 1

‎tests/contrib/openai/test_openai_v0.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,36 @@ def test_chat_completion_tool_calling(openai, openai_vcr, snapshot_tracer):
518518
)
519519

520520

521+
@pytest.mark.snapshot(
522+
token="tests.contrib.openai.test_openai.test_chat_completion_image_input",
523+
ignores=[
524+
"meta.http.useragent",
525+
"meta.openai.base_url",
526+
],
527+
)
528+
def test_chat_completion_image_input(openai, openai_vcr, snapshot_tracer):
529+
image_url = (
530+
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk"
531+
".jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
532+
)
533+
with openai_vcr.use_cassette("chat_completion_image_input.yaml"):
534+
openai.ChatCompletion.create(
535+
model="gpt-4-vision-preview",
536+
messages=[
537+
{
538+
"role": "user",
539+
"content": [
540+
{"type": "text", "text": "What’s in this image?"},
541+
{
542+
"type": "image_url",
543+
"image_url": image_url,
544+
},
545+
],
546+
}
547+
],
548+
)
549+
550+
521551
@pytest.mark.parametrize("ddtrace_config_openai", [dict(metrics_enabled=b) for b in [True, False]])
522552
def test_enable_metrics(openai, openai_vcr, ddtrace_config_openai, mock_metrics, mock_tracer):
523553
"""Ensure the metrics_enabled configuration works."""

‎tests/contrib/openai/test_openai_v1.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -568,6 +568,38 @@ def test_chat_completion_tool_calling(openai, openai_vcr, snapshot_tracer):
568568
)
569569

570570

571+
@pytest.mark.snapshot(
572+
token="tests.contrib.openai.test_openai.test_chat_completion_image_input",
573+
ignores=[
574+
"meta.http.useragent",
575+
"meta.openai.api_type",
576+
"meta.openai.api_base",
577+
],
578+
)
579+
def test_chat_completion_image_input(openai, openai_vcr, snapshot_tracer):
580+
image_url = (
581+
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk"
582+
".jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
583+
)
584+
with openai_vcr.use_cassette("chat_completion_image_input.yaml"):
585+
client = openai.OpenAI()
586+
client.chat.completions.create(
587+
model="gpt-4-vision-preview",
588+
messages=[
589+
{
590+
"role": "user",
591+
"content": [
592+
{"type": "text", "text": "What’s in this image?"},
593+
{
594+
"type": "image_url",
595+
"image_url": image_url,
596+
},
597+
],
598+
}
599+
],
600+
)
601+
602+
571603
def test_chat_completion_raw_response(openai, openai_vcr, snapshot_tracer):
572604
with snapshot_context(
573605
token="tests.contrib.openai.test_openai.test_chat_completion",

‎tests/snapshots/tests.contrib.openai.test_openai.test_chat_completion_image_input.json‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
[[
2+
{
3+
"name": "openai.request",
4+
"service": null,
5+
"resource": "createChatCompletion",
6+
"trace_id": 0,
7+
"span_id": 1,
8+
"parent_id": 0,
9+
"meta": {
10+
"_dd.p.dm": "-0",
11+
"_dd.p.tid": "6564c89d00000000",
12+
"component": "openai",
13+
"language": "python",
14+
"openai.api_base": "https://api.openai.com/v1",
15+
"openai.api_type": "open_ai",
16+
"openai.base_url": "https://api.openai.com/v1/",
17+
"openai.organization.name": "datadog-4",
18+
"openai.request.endpoint": "/v1/chat/completions",
19+
"openai.request.messages.0.content": "[{'type': 'text', 'text': 'What\u2019s in this image?'}, {'type': 'image_url', 'image_url': 'https://upload.wikimedia.org/wikipedia/c...",
20+
"openai.request.messages.0.name": "",
21+
"openai.request.messages.0.role": "user",
22+
"openai.request.method": "POST",
23+
"openai.request.model": "gpt-4-vision-preview",
24+
"openai.response.choices.0.message.content": "The image shows a serene natural landscape consisting of a wooden boardwalk that extends through",
25+
"openai.response.choices.0.message.role": "assistant",
26+
"openai.response.id": "chatcmpl-8PZ7EMif0e0dXa8yyFuSkf0kmWwKp",
27+
"openai.response.model": "gpt-4-1106-vision-preview",
28+
"openai.user.api_key": "sk-...key>",
29+
"runtime-id": "08f460eea4204da69b46486a3dc357f9"
30+
},
31+
"metrics": {
32+
"_dd.measured": 1,
33+
"_dd.top_level": 1,
34+
"_dd.tracer_kr": 1.0,
35+
"_sample_rate": 1.0,
36+
"_sampling_priority_v1": 1,
37+
"openai.organization.ratelimit.requests.remaining": 98,
38+
"openai.organization.ratelimit.tokens.remaining": 149977,
39+
"openai.response.choices_count": 1,
40+
"openai.response.created": 1701103876,
41+
"openai.response.usage.completion_tokens": 16,
42+
"openai.response.usage.prompt_tokens": 1118,
43+
"openai.response.usage.total_tokens": 1134,
44+
"process_id": 45616
45+
},
46+
"duration": 26636000,
47+
"start": 1701103773252008000
48+
}]]

0 commit comments

Comments
 (0)