From 7ce5f26bedf2c5db1bd577793b36fe6b45eccfad Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 19 Mar 2025 15:55:10 +0100 Subject: [PATCH 1/3] Bump baseline to 1.31.0 in order to have proper bucket advisory support --- .../dev-requirements-3.9.txt | 41 ++++++------- .../dev-requirements.txt | 59 ++++++++++--------- .../pyproject.toml | 8 +-- 3 files changed, 55 insertions(+), 53 deletions(-) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt index 6af20ff..53799d7 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt @@ -14,13 +14,13 @@ asgiref==3.8.1 # via opentelemetry-test-utils build==1.2.2.post1 # via pip-tools -certifi==2024.8.30 +certifi==2025.1.31 # via # httpcore # httpx -click==8.1.7 +click==8.1.8 # via pip-tools -deprecated==1.2.15 +deprecated==1.2.18 # via # opentelemetry-api # opentelemetry-semantic-conventions @@ -34,7 +34,7 @@ h11==0.14.0 # via httpcore httpcore==1.0.7 # via httpx -httpx==0.27.2 +httpx==0.28.1 # via openai idna==3.10 # via @@ -47,31 +47,31 @@ importlib-metadata==8.5.0 # opentelemetry-api iniconfig==2.0.0 # via pytest -jiter==0.7.1 +jiter==0.9.0 # via openai multidict==6.1.0 # via yarl numpy==1.24.4 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -openai==1.54.4 +openai==1.66.5 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -opentelemetry-api==1.28.2 +opentelemetry-api==1.31.0 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # opentelemetry-instrumentation # opentelemetry-sdk # opentelemetry-semantic-conventions # opentelemetry-test-utils -opentelemetry-instrumentation==0.49b2 +opentelemetry-instrumentation==0.52b0 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -opentelemetry-sdk==1.28.2 +opentelemetry-sdk==1.31.0 # via opentelemetry-test-utils -opentelemetry-semantic-conventions==0.49b2 +opentelemetry-semantic-conventions==0.52b0 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # opentelemetry-instrumentation # opentelemetry-sdk -opentelemetry-test-utils==0.49b2 +opentelemetry-test-utils==0.52b0 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) packaging==24.2 # via @@ -84,15 +84,17 @@ pluggy==1.5.0 # via pytest propcache==0.2.0 # via yarl -pydantic==2.9.2 - # via openai -pydantic-core==2.23.4 +pydantic==2.10.6 + # via + # elastic-opentelemetry-instrumentation-openai (pyproject.toml) + # openai +pydantic-core==2.27.2 # via pydantic pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.3.3 +pytest==8.3.5 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # pytest-asyncio @@ -106,14 +108,13 @@ pyyaml==6.0.2 sniffio==1.3.1 # via # anyio - # httpx # openai -tomli==2.1.0 +tomli==2.2.1 # via # build # pip-tools # pytest -tqdm==4.67.0 +tqdm==4.67.1 # via openai typing-extensions==4.12.2 # via @@ -131,9 +132,9 @@ vcrpy==6.0.2 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # pytest-vcr -wheel==0.45.0 +wheel==0.45.1 # via pip-tools -wrapt==1.16.0 +wrapt==1.17.2 # via # deprecated # elastic-opentelemetry-instrumentation-openai (pyproject.toml) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt index 8ccd85a..8c595ea 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.9.0 # via # httpx # openai @@ -14,13 +14,13 @@ asgiref==3.8.1 # via opentelemetry-test-utils build==1.2.2.post1 # via pip-tools -certifi==2024.8.30 +certifi==2025.1.31 # via # httpcore # httpx -click==8.1.7 +click==8.1.8 # via pip-tools -deprecated==1.2.15 +deprecated==1.2.18 # via # opentelemetry-api # opentelemetry-semantic-conventions @@ -34,42 +34,42 @@ h11==0.14.0 # via httpcore httpcore==1.0.7 # via httpx -httpx==0.27.2 +httpx==0.28.1 # via openai idna==3.10 # via # anyio # httpx # yarl -importlib-metadata==8.5.0 +importlib-metadata==8.6.1 # via opentelemetry-api iniconfig==2.0.0 # via pytest -jiter==0.7.1 +jiter==0.9.0 # via openai -multidict==6.1.0 +multidict==6.2.0 # via yarl -numpy==2.1.3 +numpy==2.2.4 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -openai==1.54.4 +openai==1.66.5 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -opentelemetry-api==1.28.2 +opentelemetry-api==1.31.0 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # opentelemetry-instrumentation # opentelemetry-sdk # opentelemetry-semantic-conventions # opentelemetry-test-utils -opentelemetry-instrumentation==0.49b2 +opentelemetry-instrumentation==0.52b0 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -opentelemetry-sdk==1.28.2 +opentelemetry-sdk==1.31.0 # via opentelemetry-test-utils -opentelemetry-semantic-conventions==0.49b2 +opentelemetry-semantic-conventions==0.52b0 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # opentelemetry-instrumentation # opentelemetry-sdk -opentelemetry-test-utils==0.49b2 +opentelemetry-test-utils==0.52b0 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) packaging==24.2 # via @@ -80,22 +80,24 @@ pip-tools==7.4.1 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) pluggy==1.5.0 # via pytest -propcache==0.2.0 +propcache==0.3.0 # via yarl -pydantic==2.9.2 - # via openai -pydantic-core==2.23.4 +pydantic==2.10.6 + # via + # elastic-opentelemetry-instrumentation-openai (pyproject.toml) + # openai +pydantic-core==2.27.2 # via pydantic pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.3.3 +pytest==8.3.5 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # pytest-asyncio # pytest-vcr -pytest-asyncio==0.24.0 +pytest-asyncio==0.25.3 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) pytest-vcr==1.0.2 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) @@ -104,14 +106,13 @@ pyyaml==6.0.2 sniffio==1.3.1 # via # anyio - # httpx # openai -tomli==2.1.0 +tomli==2.2.1 # via # build # pip-tools # pytest -tqdm==4.67.0 +tqdm==4.67.1 # via openai typing-extensions==4.12.2 # via @@ -122,21 +123,21 @@ typing-extensions==4.12.2 # opentelemetry-sdk # pydantic # pydantic-core -urllib3==2.2.3 +urllib3==2.3.0 # via vcrpy -vcrpy==6.0.2 +vcrpy==7.0.0 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # pytest-vcr -wheel==0.45.0 +wheel==0.45.1 # via pip-tools -wrapt==1.16.0 +wrapt==1.17.2 # via # deprecated # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # opentelemetry-instrumentation # vcrpy -yarl==1.17.2 +yarl==1.18.3 # via vcrpy zipp==3.21.0 # via importlib-metadata diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml index 84f3f35..59a25a6 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml @@ -25,10 +25,10 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - # 1.28.2 is required for *robust* Events API/SDK - "opentelemetry-api ~= 1.28", - "opentelemetry-instrumentation ~= 0.49b2", - "opentelemetry-semantic-conventions ~= 0.49b2", + # 1.31.0 is required for proper histogram bucket advisory support + "opentelemetry-api ~= 1.31", + "opentelemetry-instrumentation ~= 0.52b0", + "opentelemetry-semantic-conventions ~= 0.52b0", "wrapt >= 1.0.0, < 2.0.0", ] From 2f02a7a351bdc691a5147ba2a29beb2089e25fa0 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 19 Mar 2025 16:05:18 +0100 Subject: [PATCH 2/3] openai: use explicit_bucket_boundaries_advisory to use genai suggested buckets --- .../instrumentation/openai/__init__.py | 23 +++++++-- .../instrumentation/openai/metrics.py | 49 +++++++++++++++++++ .../tests/conftest.py | 9 ++++ .../tests/utils.py | 12 +++-- 4 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/metrics.py diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index fddc770..2042ef9 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -36,14 +36,18 @@ _send_log_events_from_messages, _span_name_from_attributes, ) +from opentelemetry.instrumentation.openai.metrics import ( + _GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS, + _GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS, +) from opentelemetry.instrumentation.openai.package import _instruments from opentelemetry.instrumentation.openai.version import __version__ from opentelemetry.instrumentation.openai.wrappers import StreamWrapper from opentelemetry.instrumentation.utils import unwrap from opentelemetry.metrics import get_meter from opentelemetry.semconv._incubating.metrics.gen_ai_metrics import ( - create_gen_ai_client_operation_duration, - create_gen_ai_client_token_usage, + GEN_AI_CLIENT_OPERATION_DURATION, + GEN_AI_CLIENT_TOKEN_USAGE, ) from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE from opentelemetry.semconv.schemas import Schemas @@ -94,8 +98,19 @@ def _instrument(self, **kwargs): event_logger_provider = kwargs.get("event_logger_provider") self.event_logger = get_event_logger(__name__, event_logger_provider) - self.token_usage_metric = create_gen_ai_client_token_usage(self.meter) - self.operation_duration_metric = create_gen_ai_client_operation_duration(self.meter) + self.token_usage_metric = self.meter.create_histogram( + name=GEN_AI_CLIENT_TOKEN_USAGE, + description="Measures number of input and output tokens used", + unit="{token}", + explicit_bucket_boundaries_advisory=_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS, + ) + + self.operation_duration_metric = self.meter.create_histogram( + name=GEN_AI_CLIENT_OPERATION_DURATION, + description="GenAI operation duration", + unit="s", + explicit_bucket_boundaries_advisory=_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS, + ) register_post_import_hook(self._patch, "openai") diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/metrics.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/metrics.py new file mode 100644 index 0000000..427eaa0 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/metrics.py @@ -0,0 +1,49 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS = [ + 0.01, + 0.02, + 0.04, + 0.08, + 0.16, + 0.32, + 0.64, + 1.28, + 2.56, + 5.12, + 10.24, + 20.48, + 40.96, + 81.92, +] + +_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS = [ + 1, + 4, + 16, + 64, + 256, + 1024, + 4096, + 16384, + 65536, + 262144, + 1048576, + 4194304, + 16777216, + 67108864, +] diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py index b4e7d8a..d8320d3 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py @@ -26,6 +26,10 @@ from opentelemetry._events import set_event_logger_provider from opentelemetry._logs import set_logger_provider from opentelemetry.instrumentation.openai import OpenAIInstrumentor +from opentelemetry.instrumentation.openai.metrics import ( + _GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS, + _GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS, +) from opentelemetry.metrics import Histogram from opentelemetry.sdk._events import EventLoggerProvider from opentelemetry.sdk._logs import LoggerProvider @@ -219,6 +223,7 @@ def assert_operation_duration_metric( max_data_point=max_data_point, min_data_point=min_data_point, attributes={**default_attributes, **attributes}, + explicit_bounds=tuple(_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS), ), ], est_value_delta=0.25, @@ -245,6 +250,7 @@ def assert_error_operation_duration_metric( max_data_point=data_point, min_data_point=data_point, attributes={**default_attributes, **attributes}, + explicit_bounds=tuple(_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS), ), ], est_value_delta=value_delta, @@ -272,6 +278,7 @@ def assert_token_usage_input_metric( max_data_point=input_data_point, min_data_point=input_data_point, attributes={**default_attributes, **attributes}, + explicit_bounds=tuple(_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS), ), ], ) @@ -320,6 +327,7 @@ def assert_token_usage_metric( max_data_point=max_input, min_data_point=min_input, attributes={**default_attributes, **attributes, "gen_ai.token.type": "input"}, + explicit_bounds=tuple(_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS), ), create_histogram_data_point( count=count, @@ -327,6 +335,7 @@ def assert_token_usage_metric( max_data_point=max_output, min_data_point=min_output, attributes={**default_attributes, **attributes, "gen_ai.token.type": "output"}, + explicit_bounds=tuple(_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS), ), ], ) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py index fb5b121..386a8cc 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py @@ -86,7 +86,11 @@ def is_data_points_equal( ): return False - return values_diff <= est_value_delta and expected_data_point.attributes == dict(data_point.attributes) + return ( + values_diff <= est_value_delta + and expected_data_point.attributes == dict(data_point.attributes) + and expected_data_point.explicit_bounds == data_point.explicit_bounds + ) def assert_data_point_expected( @@ -115,7 +119,9 @@ def create_number_data_point(value, attributes): ) -def create_histogram_data_point(sum_data_point, count, max_data_point, min_data_point, attributes): +def create_histogram_data_point( + sum_data_point, count, max_data_point, min_data_point, attributes, explicit_bounds=None +): return HistogramDataPoint( count=count, sum=sum_data_point, @@ -125,7 +131,7 @@ def create_histogram_data_point(sum_data_point, count, max_data_point, min_data_ start_time_unix_nano=0, time_unix_nano=0, bucket_counts=[], - explicit_bounds=[], + explicit_bounds=explicit_bounds or [], ) From 177b37c2d3f75936c0d199bdaf387f25d38ac6bd Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 19 Mar 2025 16:32:09 +0100 Subject: [PATCH 3/3] Keep an older version of httpx for baseline testing --- .../dev-requirements-3.9.txt | 2 +- .../dev-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt index 53799d7..0ca02dd 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements-3.9.txt @@ -34,7 +34,7 @@ h11==0.14.0 # via httpcore httpcore==1.0.7 # via httpx -httpx==0.28.1 +httpx==0.27.2 # via openai idna==3.10 # via diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt index 8c595ea..38630c2 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt @@ -34,7 +34,7 @@ h11==0.14.0 # via httpcore httpcore==1.0.7 # via httpx -httpx==0.28.1 +httpx==0.27.2 # via openai idna==3.10 # via