Fix all tests

Author: DylanRussell

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py

@@ -150,7 +150,7 @@ def handle_response(
                     )
                 self.event_logger.emit(
                     create_operation_details_event(
-                        api_endpoint = api_endpoint,
+                        api_endpoint=api_endpoint,
                         params=params,
                         capture_content=self.capture_content,
                         response=response,
@@ -225,13 +225,30 @@ def generate_content(
         | prediction_service_v1beta1.GenerateContentResponse
     ):
         if self.sem_conv_opt_in_mode == _StabilityMode.DEFAULT:
-            _instrumentation = self._with_default_instrumentation
+            with self._with_default_instrumentation(
+                instance, args, kwargs
+            ) as handle_response:
+                response = wrapped(*args, **kwargs)
+                handle_response(response)
+                return response
         else:
-            _instrumentation = self._with_new_instrumentation
-        with _instrumentation(instance, args, kwargs) as handle_response:
-            response = wrapped(*args, **kwargs)
-            handle_response(response)
-            return response
+            with self._with_new_instrumentation(
+                instance, args, kwargs
+            ) as handle_response:
+                try:
+                    response = wrapped(*args, **kwargs)
+                except Exception as e:
+                    self.event_logger.emit(
+                        create_operation_details_event(
+                            params=_extract_params(*args, **kwargs),
+                            response=None,
+                            capture_content=self.capture_content,
+                            api_endpoint=instance.api_endpoint,
+                        )
+                    )
+                    raise e
+                handle_response(response)
+                return response
 
     async def agenerate_content(
         self,
@@ -251,10 +268,27 @@ async def agenerate_content(
         | prediction_service_v1beta1.GenerateContentResponse
     ):
         if self.sem_conv_opt_in_mode == _StabilityMode.DEFAULT:
-            _instrumentation = self._with_default_instrumentation
+            with self._with_default_instrumentation(
+                instance, args, kwargs
+            ) as handle_response:
+                response = await wrapped(*args, **kwargs)
+                handle_response(response)
+                return response
         else:
-            _instrumentation = self._with_new_instrumentation
-        with _instrumentation(instance, args, kwargs) as handle_response:
-            response = await wrapped(*args, **kwargs)
-            handle_response(response)
-            return response
+            with self._with_new_instrumentation(
+                instance, args, kwargs
+            ) as handle_response:
+                try:
+                    response = await wrapped(*args, **kwargs)
+                except Exception as e:
+                    self.event_logger.emit(
+                        create_operation_details_event(
+                            params=_extract_params(*args, **kwargs),
+                            response=None,
+                            capture_content=self.capture_content,
+                            api_endpoint=instance.api_endpoint,
+                        )
+                    )
+                    raise e
+                handle_response(response)
+                return response

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py

@@ -129,6 +129,8 @@ def get_genai_request_attributes(
             generation_config.temperature
         )
     if "top_p" in generation_config:
+        # There is also a top_k parameter ( The maximum number of tokens to consider when sampling.),
+        # but no semconv yet exists for it.
         attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_P] = (
             generation_config.top_p
         )
@@ -144,14 +146,28 @@ def get_genai_request_attributes(
         attributes[GenAIAttributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] = (
             generation_config.frequency_penalty
         )
-    if "seed" in generation_config and use_latest_semconvs:
-        attributes[GenAIAttributes.GEN_AI_REQUEST_SEED] = (
-            generation_config.seed
-        )
     if "stop_sequences" in generation_config:
         attributes[GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES] = (
             generation_config.stop_sequences
         )
+    if use_latest_semconvs:
+        if "seed" in generation_config:
+            attributes[GenAIAttributes.GEN_AI_REQUEST_SEED] = (
+                generation_config.seed
+            )
+        if "candidate_count" in generation_config:
+            attributes[GenAIAttributes.GEN_AI_REQUEST_CHOICE_COUNT] = (
+                generation_config.candidate_count
+            )
+        if "response_mime_type" in generation_config:
+            if generation_config.response_mime_type == "text/plain":
+                attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = "text"
+            elif generation_config.response_mime_type == "application/json":
+                attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = "json"
+            else:
+                attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = (
+                    generation_config.response_mime_type
+                )
 
     return attributes
 
@@ -164,8 +180,6 @@ def get_genai_response_attributes(
         _map_finish_reason(candidate.finish_reason)
         for candidate in response.candidates
     ]
-    # TODO: add gen_ai.response.id once available in the python client
-    # https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3246
     return {
         GenAIAttributes.GEN_AI_RESPONSE_MODEL: response.model_version,
         GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS: finish_reasons,
@@ -262,14 +276,16 @@ def create_operation_details_event(
     *,
     api_endpoint: str,
     response: prediction_service.GenerateContentResponse
-    | prediction_service_v1beta1.GenerateContentResponse,
+    | prediction_service_v1beta1.GenerateContentResponse
+    | None,
     params: GenerateContentParams,
     capture_content: bool,
 ) -> Event:
     event = Event(name="gen_ai.client.inference.operation.details")
     attributes: dict[str, AnyValue] = {
         **get_genai_request_attributes(True, params),
         **get_server_attributes(api_endpoint),
+        **(get_genai_response_attributes(response) if response else {}),
     }
     event.attributes = attributes
     if not capture_content:
@@ -287,7 +303,7 @@ def create_operation_details_event(
         attributes["gen_ai.input.messages"] = [
             _convert_content_to_message(content) for content in params.contents
         ]
-    if response.candidates:
+    if response and response.candidates:
         attributes["gen_ai.output.messages"] = (
             _convert_response_to_output_messages(response)
         )
@@ -310,13 +326,13 @@ def _convert_content_to_message(content: content.Content) -> dict:
     message = {}
     message["role"] = content.role
     message["parts"] = []
-    for part in content.parts:
+    for idx, part in enumerate(content.parts):
         if "function_response" in part:
             part = part.function_response
             message["parts"].append(
                 {
                     "type": "tool_call_response",
-                    "id": part.id,
+                    "id": f"{part.name}_{idx}",
                     "response": json_format.MessageToDict(part._pb.response),
                 }
             )
@@ -325,9 +341,8 @@ def _convert_content_to_message(content: content.Content) -> dict:
             message["parts"].append(
                 {
                     "type": "tool_call",
-                    "id": part.id,
+                    "id": f"{part.name}_{idx}",
                     "name": part.name,
-                    # TODO: support partial_args/streaming  here?
                     "response": json_format.MessageToDict(
                         part._pb.args,
                     ),

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py

@@ -10,7 +10,6 @@
     Generator,
     Mapping,
     MutableMapping,
-    Optional,
     Protocol,
     TypeVar,
 )
@@ -33,6 +32,7 @@
 
 from opentelemetry.instrumentation._semconv import (
     OTEL_SEMCONV_STABILITY_OPT_IN,
+    _OpenTelemetrySemanticConventionStability,
 )
 from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
 from opentelemetry.instrumentation.vertexai.utils import (
@@ -122,13 +122,17 @@ def vertexai_init(vcr: VCR) -> None:
 def instrument_no_content(
     tracer_provider, event_logger_provider, meter_provider, request
 ):
+    # Reset global state..
+    _OpenTelemetrySemanticConventionStability._initialized = False
     os.environ.update(
         {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "False"}
     )
     if request.param:
         os.environ.update(
             {OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental"}
         )
+    else:
+        os.environ.update({OTEL_SEMCONV_STABILITY_OPT_IN: "stable"})
 
     instrumentor = VertexAIInstrumentor()
     instrumentor.instrument(
@@ -150,13 +154,17 @@ def instrument_no_content(
 def instrument_with_content(
     tracer_provider, event_logger_provider, meter_provider, request
 ):
+    # Reset global state..
+    _OpenTelemetrySemanticConventionStability._initialized = False
     os.environ.update(
         {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"}
     )
     if request.param:
         os.environ.update(
             {OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental"}
         )
+    else:
+        os.environ.update({OTEL_SEMCONV_STABILITY_OPT_IN: "stable"})
     instrumentor = VertexAIInstrumentor()
     instrumentor.instrument(
         tracer_provider=tracer_provider,
@@ -304,7 +312,6 @@ def __call__(self): ...
 def fixture_generate_content(
     request: pytest.FixtureRequest,
     vcr: VCR,
-    cassette_name: Optional[str] = None,
 ) -> Generator[GenerateContentFixture, None, None]:
     """This fixture parameterizes tests that use it to test calling both
     GenerativeModel.generate_content() and GenerativeModel.generate_content_async().
@@ -322,6 +329,6 @@ def wrapper(model: GenerativeModel, *args, **kwargs) -> None:
         return model.generate_content(*args, **kwargs)
 
     with vcr.use_cassette(
-        cassette_name or request.node.originalname, allow_playback_repeats=True
+        request.node.originalname, allow_playback_repeats=True
     ):
         yield wrapper