From 082ee909c8f052259e572974a77b41570f0a3fd7 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 3 Mar 2025 09:43:22 +0100 Subject: [PATCH] openai: handle message with developer role Send an system event with an explicit role attribute set to developer in the body --- .../instrumentation/openai/helpers.py | 4 +- ...test_chat_with_developer_role_message.yaml | 138 ++++++++++++++++++ .../tests/test_chat_completions.py | 72 +++++++++ 3 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/test_chat_with_developer_role_message.yaml diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py index 7cee671..02b53a6 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py @@ -255,7 +255,9 @@ def _send_log_events_from_messages( content = message.get("content") if content: body["content"] = content - if message["role"] == "system": + if message["role"] == "system" or message["role"] == "developer": + if message["role"] == "developer": + body["role"] = message["role"] event = Event(name=EVENT_GEN_AI_SYSTEM_MESSAGE, body=body, attributes=attributes) event_logger.emit(event) elif message["role"] == "user": diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/test_chat_with_developer_role_message.yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/test_chat_with_developer_role_message.yaml new file mode 100644 index 0000000..f1b3d48 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/test_chat_with_developer_role_message.yaml @@ -0,0 +1,138 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "developer", + "content": "You are a friendly assistant" + }, + { + "role": "user", + "content": "Answer in up to 3 words: Which ocean contains Bouvet Island?" + } + ], + "model": "gpt-4o-mini" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '197' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.54.4 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.54.4 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.8 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-B6vdHtqgT6rj4cj7itn9bNlaUlqHg", + "object": "chat.completion", + "created": 1740991207, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Atlantic Ocean.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 4, + "total_tokens": 35, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_06737a9306" + } + headers: + CF-RAY: + - 91a7d3c46d41eda3-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 03 Mar 2025 08:40:08 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '794' + openai-organization: test_openai_org_id + openai-processing-ms: + - '169' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199960' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 12ms + x-request-id: + - req_70b661026c20278c4d79838fbba06bde + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py index e83a5c5..a785eef 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py @@ -130,6 +130,78 @@ def test_chat(default_openai_env, trace_exporter, metrics_reader, logs_exporter) ) +@pytest.mark.vcr() +def test_chat_with_developer_role_message(default_openai_env, trace_exporter, metrics_reader, logs_exporter): + client = openai.OpenAI() + + messages = [ + { + "role": "developer", + "content": "You are a friendly assistant", + }, + { + "role": "user", + "content": TEST_CHAT_INPUT, + }, + ] + + chat_completion = client.chat.completions.create(model=TEST_CHAT_MODEL, messages=messages) + + assert chat_completion.choices[0].message.content == "Atlantic Ocean." + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {TEST_CHAT_MODEL}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + address, port = address_and_port(client) + assert dict(span.attributes) == { + GEN_AI_OPENAI_RESPONSE_SERVICE_TIER: "default", + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: TEST_CHAT_MODEL, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: "chatcmpl-B6vdHtqgT6rj4cj7itn9bNlaUlqHg", + GEN_AI_RESPONSE_MODEL: TEST_CHAT_RESPONSE_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS: ("stop",), + GEN_AI_USAGE_INPUT_TOKENS: 31, + GEN_AI_USAGE_OUTPUT_TOKENS: 4, + SERVER_ADDRESS: address, + SERVER_PORT: port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 3 + log_records = logrecords_from_logs(logs) + system_message, user_message, choice = log_records + assert dict(system_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.system.message"} + assert dict(system_message.body) == {"role": "developer"} + + assert dict(user_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert dict(user_message.body) == {} + + assert_stop_log_record(choice) + + operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: TEST_CHAT_MODEL, + GEN_AI_RESPONSE_MODEL: "gpt-4o-mini-2024-07-18", + } + assert_operation_duration_metric( + client, "chat", operation_duration_metric, attributes=attributes, min_data_point=0.006761051714420319 + ) + assert_token_usage_metric( + client, + "chat", + token_usage_metric, + attributes=attributes, + input_data_point=span.attributes[GEN_AI_USAGE_INPUT_TOKENS], + output_data_point=span.attributes[GEN_AI_USAGE_OUTPUT_TOKENS], + ) + + @pytest.mark.skipif(OPENAI_VERSION < (1, 35, 0), reason="service tier added in 1.35.0") @pytest.mark.vcr() def test_chat_all_the_client_options(default_openai_env, trace_exporter, metrics_reader, logs_exporter):