Skip to content

Commit 231d26c

Browse files
Liudmila Molkovaxrmx
Liudmila Molkova
and
xrmx
authored
Provide advisory histogram boundaries when creating OpenAI metrics (open-telemetry#3225)
* small fixes in OpenAI examples * up * add comment * leverage histogram bucket advice in 1.30 * Update instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_metrics.py * Update instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_metrics.py * up --------- Co-authored-by: Riccardo Magliocchetti <[email protected]>
1 parent c0bc2c9 commit 231d26c

File tree

8 files changed

+100
-116
lines changed

8 files changed

+100
-116
lines changed

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/README.rst‎

Lines changed: 0 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -75,48 +75,6 @@ To uninstrument clients, call the uninstrument method:
7575
# Uninstrument all clients
7676
OpenAIInstrumentor().uninstrument()
7777
78-
Bucket Boundaries
79-
-----------------
80-
81-
This section describes the explicit bucket boundaries for metrics such as token usage and operation duration, and guides users to create Views to implement them according to the semantic conventions.
82-
83-
The bucket boundaries are defined as follows:
84-
85-
- For `gen_ai.client.token.usage`: [1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864]
86-
- For `gen_ai.client.operation.duration`: [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]
87-
88-
To implement these bucket boundaries, you can create Views in your OpenTelemetry SDK setup. Here is an example:
89-
90-
.. code-block:: python
91-
92-
from opentelemetry.sdk.metrics import MeterProvider, View
93-
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
94-
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
95-
from opentelemetry.sdk.metrics.aggregation import ExplicitBucketHistogramAggregation
96-
97-
views = [
98-
View(
99-
instrument_name="gen_ai.client.token.usage",
100-
aggregation=ExplicitBucketHistogramAggregation([1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864]),
101-
),
102-
View(
103-
instrument_name="gen_ai.client.operation.duration",
104-
aggregation=ExplicitBucketHistogramAggregation([0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]),
105-
),
106-
]
107-
108-
metric_exporter = OTLPMetricExporter(endpoint="http://localhost:4317")
109-
metric_reader = PeriodicExportingMetricReader(metric_exporter)
110-
provider = MeterProvider(
111-
metric_readers=[metric_reader],
112-
views=views
113-
)
114-
115-
from opentelemetry.sdk.metrics import set_meter_provider
116-
set_meter_provider(provider)
117-
118-
For more details, refer to the `OpenTelemetry GenAI Metrics documentation <https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/>`_.
119-
12078
References
12179
----------
12280
* `OpenTelemetry OpenAI Instrumentation <https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation-genai/openai.html>`_

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/examples/manual/requirements.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
openai~=1.57.3
22

3-
opentelemetry-sdk~=1.29.0
4-
opentelemetry-exporter-otlp-proto-grpc~=1.29.0
5-
opentelemetry-instrumentation-openai-v2~=2.0b0
3+
opentelemetry-sdk~=1.30.0
4+
opentelemetry-exporter-otlp-proto-grpc~=1.30.0
5+
opentelemetry-instrumentation-openai-v2~=2.2b0

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/examples/zero-code/requirements.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
openai~=1.57.3
22

3-
opentelemetry-sdk~=1.29.0
4-
opentelemetry-exporter-otlp-proto-grpc~=1.29.0
5-
opentelemetry-distro~=0.50b0
6-
opentelemetry-instrumentation-openai-v2~=2.0b0
3+
opentelemetry-sdk~=1.30.0
4+
opentelemetry-exporter-otlp-proto-grpc~=1.30.0
5+
opentelemetry-distro~=0.51b0
6+
opentelemetry-instrumentation-openai-v2~=2.2b0

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/pyproject.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ classifiers = [
2626
"Programming Language :: Python :: 3.13",
2727
]
2828
dependencies = [
29-
"opentelemetry-api ~= 1.28",
30-
"opentelemetry-instrumentation ~= 0.49b0",
31-
"opentelemetry-semantic-conventions ~= 0.49b0"
29+
"opentelemetry-api ~= 1.30",
30+
"opentelemetry-instrumentation ~= 0.51b0",
31+
"opentelemetry-semantic-conventions ~= 0.51b0"
3232
]
3333

3434
[project.optional-dependencies]

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/instruments.py‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,52 @@
1+
from opentelemetry.metrics import Histogram, Meter
12
from opentelemetry.semconv._incubating.metrics import gen_ai_metrics
23

4+
_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS = [
5+
0.01,
6+
0.02,
7+
0.04,
8+
0.08,
9+
0.16,
10+
0.32,
11+
0.64,
12+
1.28,
13+
2.56,
14+
5.12,
15+
10.24,
16+
20.48,
17+
40.96,
18+
81.92,
19+
]
20+
21+
_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS = [
22+
1,
23+
4,
24+
16,
25+
64,
26+
256,
27+
1024,
28+
4096,
29+
16384,
30+
65536,
31+
262144,
32+
1048576,
33+
4194304,
34+
16777216,
35+
67108864,
36+
]
37+
338

439
class Instruments:
5-
def __init__(self, meter):
6-
self.operation_duration_histogram = (
7-
gen_ai_metrics.create_gen_ai_client_operation_duration(meter)
40+
def __init__(self, meter: Meter):
41+
self.operation_duration_histogram: Histogram = meter.create_histogram(
42+
name=gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION,
43+
description="GenAI operation duration",
44+
unit="s",
45+
explicit_bucket_boundaries_advisory=_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS,
846
)
9-
self.token_usage_histogram = (
10-
gen_ai_metrics.create_gen_ai_client_token_usage(meter)
47+
self.token_usage_histogram: Histogram = meter.create_histogram(
48+
name=gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE,
49+
description="Measures number of input and output tokens used",
50+
unit="{token}",
51+
explicit_bucket_boundaries_advisory=_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS,
1152
)

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/test-requirements-0.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ pytest==7.4.4
88
pytest-vcr==1.0.2
99
pytest-asyncio==0.21.0
1010
wrapt==1.16.0
11-
opentelemetry-exporter-otlp-proto-http~=1.28
12-
opentelemetry-api==1.28 # when updating, also update in pyproject.toml
13-
opentelemetry-sdk==1.28 # when updating, also update in pyproject.toml
14-
opentelemetry-semantic-conventions==0.49b0 # when updating, also update in pyproject.toml
11+
opentelemetry-exporter-otlp-proto-http~=1.30
12+
opentelemetry-api==1.30 # when updating, also update in pyproject.toml
13+
opentelemetry-sdk==1.30 # when updating, also update in pyproject.toml
14+
opentelemetry-semantic-conventions==0.51b0 # when updating, also update in pyproject.toml
1515

1616
-e instrumentation-genai/opentelemetry-instrumentation-openai-v2

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py‎

Lines changed: 0 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,11 @@
1818
SimpleLogRecordProcessor,
1919
)
2020
from opentelemetry.sdk.metrics import (
21-
Histogram,
2221
MeterProvider,
2322
)
2423
from opentelemetry.sdk.metrics.export import (
2524
InMemoryMetricReader,
2625
)
27-
from opentelemetry.sdk.metrics.view import (
28-
ExplicitBucketHistogramAggregation,
29-
View,
30-
)
3126
from opentelemetry.sdk.trace import TracerProvider
3227
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
3328
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
@@ -72,55 +67,8 @@ def fixture_event_logger_provider(log_exporter):
7267

7368
@pytest.fixture(scope="function", name="meter_provider")
7469
def fixture_meter_provider(metric_reader):
75-
token_usage_histogram_view = View(
76-
instrument_type=Histogram,
77-
instrument_name="gen_ai.client.token.usage",
78-
aggregation=ExplicitBucketHistogramAggregation(
79-
boundaries=[
80-
1,
81-
4,
82-
16,
83-
64,
84-
256,
85-
1024,
86-
4096,
87-
16384,
88-
65536,
89-
262144,
90-
1048576,
91-
4194304,
92-
16777216,
93-
67108864,
94-
]
95-
),
96-
)
97-
98-
duration_histogram_view = View(
99-
instrument_type=Histogram,
100-
instrument_name="gen_ai.client.operation.duration",
101-
aggregation=ExplicitBucketHistogramAggregation(
102-
boundaries=[
103-
0.01,
104-
0.02,
105-
0.04,
106-
0.08,
107-
0.16,
108-
0.32,
109-
0.64,
110-
1.28,
111-
2.56,
112-
5.12,
113-
10.24,
114-
20.48,
115-
40.96,
116-
81.92,
117-
]
118-
),
119-
)
120-
12170
meter_provider = MeterProvider(
12271
metric_readers=[metric_reader],
123-
views=[token_usage_histogram_view, duration_histogram_view],
12472
)
12573

12674
return meter_provider

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_metrics.py‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,39 @@
88
)
99
from opentelemetry.semconv._incubating.metrics import gen_ai_metrics
1010

11+
_DURATION_BUCKETS = (
12+
0.01,
13+
0.02,
14+
0.04,
15+
0.08,
16+
0.16,
17+
0.32,
18+
0.64,
19+
1.28,
20+
2.56,
21+
5.12,
22+
10.24,
23+
20.48,
24+
40.96,
25+
81.92,
26+
)
27+
_TOKEN_USAGE_BUCKETS = (
28+
1,
29+
4,
30+
16,
31+
64,
32+
256,
33+
1024,
34+
4096,
35+
16384,
36+
65536,
37+
262144,
38+
1048576,
39+
4194304,
40+
16777216,
41+
67108864,
42+
)
43+
1144

1245
def assert_all_metric_attributes(data_point):
1346
assert GenAIAttributes.GEN_AI_OPERATION_NAME in data_point.attributes
@@ -77,8 +110,11 @@ def test_chat_completion_metrics(
77110
None,
78111
)
79112
assert duration_metric is not None
80-
assert duration_metric.data.data_points[0].sum > 0
81-
assert_all_metric_attributes(duration_metric.data.data_points[0])
113+
114+
duration_point = duration_metric.data.data_points[0]
115+
assert duration_point.sum > 0
116+
assert_all_metric_attributes(duration_point)
117+
assert duration_point.explicit_bounds == _DURATION_BUCKETS
82118

83119
token_usage_metric = next(
84120
(
@@ -101,7 +137,8 @@ def test_chat_completion_metrics(
101137
)
102138
assert input_token_usage is not None
103139
assert input_token_usage.sum == 12
104-
# assert against buckets [1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864]
140+
141+
assert input_token_usage.explicit_bounds == _TOKEN_USAGE_BUCKETS
105142
assert input_token_usage.bucket_counts[2] == 1
106143
assert_all_metric_attributes(input_token_usage)
107144

0 commit comments

Comments
 (0)