diff --git a/newrelic/config.py b/newrelic/config.py index d3dd127645..8bce65b9e2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2009,6 +2009,8 @@ def _process_module_builtin_defaults(): ) _process_module_definition("openai._base_client", "newrelic.hooks.mlmodel_openai", "instrument_openai_base_client") + _process_module_definition("google.genai.models", "newrelic.hooks.mlmodel_gemini", "instrument_genai_models") + _process_module_definition( "asyncio.base_events", "newrelic.hooks.coroutines_asyncio", "instrument_asyncio_base_events" ) diff --git a/newrelic/hooks/mlmodel_gemini.py b/newrelic/hooks/mlmodel_gemini.py new file mode 100644 index 0000000000..d2be0cb0e5 --- /dev/null +++ b/newrelic/hooks/mlmodel_gemini.py @@ -0,0 +1,605 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import uuid + +import google + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.core.config import global_settings + +GEMINI_VERSION = get_package_version("google-genai") +EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE = ( + "Exception occurred in Gemini instrumentation: While reporting an exception " + "in Gemini, another exception occurred. Report this issue to New Relic " + "Support.\n " +) +RECORD_EVENTS_FAILURE_LOG_MESSAGE = ( + "Exception occurred in Gemini instrumentation: Failed to record LLM events. " + "Please report this issue to New Relic Support.\n " +) + +_logger = logging.getLogger(__name__) + + +def wrap_embed_content_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + # Obtain attributes to be stored on embedding events regardless of whether we hit an error + embedding_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/embedding/Gemini") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = wrapped(*args, **kwargs) + except Exception as exc: + _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft) + return response + + +async def wrap_embed_content_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + # Obtain attributes to be stored on embedding events regardless of whether we hit an error + embedding_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/embedding/Gemini") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = await wrapped(*args, **kwargs) + except Exception as exc: + _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft) + return response + + +def _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc): + settings = transaction.settings or global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + + notice_error_attributes = {} + try: + # We key directly into the kwargs dict here so that we can raise a KeyError if "contents" is not available + embedding_content = kwargs["contents"] + # embedding_content could be a list, so we typecast it to a string + embedding_content = str(embedding_content) + model = kwargs.get("model") + + notice_error_attributes = { + "http.statusCode": getattr(exc, "code", None), + "error.message": getattr(exc, "message", None), + "error.code": getattr(exc, "status", None), # ex: 'NOT_FOUND' + "embedding_id": embedding_id, + } + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE, exc_info=True) + + message = notice_error_attributes.pop("error.message", None) + if message: + exc._nr_message = message + + ft.notice_error(attributes=notice_error_attributes) + # Exit the trace now so that the duration is calculated. + ft.__exit__(*sys.exc_info()) + + try: + error_embedding_dict = { + "id": embedding_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(model, embedding_content) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "request.model": model, + "vendor": "gemini", + "ingest_source": "Python", + "duration": ft.duration * 1000, + "error": True, + } + if settings.ai_monitoring.record_content.enabled: + error_embedding_dict["input"] = embedding_content + + error_embedding_dict.update(_get_llm_attributes(transaction)) + transaction.record_custom_event("LlmEmbedding", error_embedding_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + +def _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft): + settings = transaction.settings or global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + try: + # We key directly into the kwargs dict here so that we can raise a KeyError if "contents" is not available + embedding_content = kwargs["contents"] + # embedding_content could be a list, so we typecast it to a string + embedding_content = str(embedding_content) + request_model = kwargs.get("model") + + full_embedding_response_dict = { + "id": embedding_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(request_model, embedding_content) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "request.model": request_model, + "duration": ft.duration * 1000, + "vendor": "gemini", + "ingest_source": "Python", + } + if settings.ai_monitoring.record_content.enabled: + full_embedding_response_dict["input"] = embedding_content + + full_embedding_response_dict.update(_get_llm_attributes(transaction)) + + transaction.record_custom_event("LlmEmbedding", full_embedding_response_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + +def _get_llm_attributes(transaction): + """Returns llm.* custom attributes off of the transaction.""" + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + + return llm_metadata_dict + + +def wrap_generate_content_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/Gemini") + ft.__enter__() + + linking_metadata = get_trace_linking_metadata() + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _record_generation_error(transaction, settings, linking_metadata, completion_id, kwargs, ft, exc) + raise + + ft.__exit__(None, None, None) + + _handle_generation_success(transaction, settings, linking_metadata, completion_id, kwargs, ft, return_val) + + return return_val + + +async def wrap_generate_content_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/Gemini") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = await wrapped(*args, **kwargs) + except Exception as exc: + _record_generation_error(transaction, settings, linking_metadata, completion_id, kwargs, ft, exc) + raise + + ft.__exit__(None, None, None) + + _handle_generation_success(transaction, settings, linking_metadata, completion_id, kwargs, ft, return_val) + + return return_val + + +def _record_generation_error(transaction, settings, linking_metadata, completion_id, kwargs, ft, exc): + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + + # If generate_content was called directly, "contents" should hold a string with just the user input. + # If send_message was called, "contents" should hold a list containing the user input & role. + # When send_message is called multiple times within a chat conversation, "contents" will hold chat history with + # multiple lists to capture each input to the LLM (only inputs and not responses) + messages = kwargs.get("contents") + if isinstance(messages, str): + input_message = messages + else: + try: + input_message = messages[-1] + except Exception: + input_message = None + _logger.warning( + "Unable to parse input message to Gemini LLM. Message content and role will be omitted from " + "corresponding LlmChatCompletionMessage event. " + ) + + # Extract the input message content and role from the input message if it exists + input_message_content, input_role = _parse_input_message(input_message) if input_message else (None, None) + + # Extract data from generation config object + request_temperature, request_max_tokens = _extract_generation_config(kwargs) + + # Extract model and prompt token count + request_model = kwargs.get("model") + prompt_tokens = ( + settings.ai_monitoring.llm_token_count_callback(request_model, input_message_content) + if settings.ai_monitoring.llm_token_count_callback and input_message_content + else None + ) + + # Prepare error attributes + notice_error_attributes = { + "http.statusCode": getattr(exc, "code", None), + "error.message": getattr(exc, "message", None), + "error.code": getattr(exc, "status", None), # ex: 'NOT_FOUND' + "completion_id": completion_id, + } + + # Override the default message if it is not empty. + message = notice_error_attributes.pop("error.message", None) + if message: + exc._nr_message = message + + ft.notice_error(attributes=notice_error_attributes) + # Stop the span now so we compute the duration before we create the events. + ft.__exit__(*sys.exc_info()) + + try: + error_chat_completion_dict = { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "response.number_of_messages": len(messages), + "request.model": request_model, + "request.temperature": request_temperature, + "request.max_tokens": request_max_tokens, + "response.usage.prompt_tokens": prompt_tokens, + # In error cases, we do not have a response to calculate completion tokens on so omit them from the LLM + # event and set total tokens to the prompt tokens value + "response.usage.total_tokens": prompt_tokens, + "vendor": "gemini", + "ingest_source": "Python", + "duration": ft.duration * 1000, + "error": True, + } + llm_metadata = _get_llm_attributes(transaction) + error_chat_completion_dict.update(llm_metadata) + transaction.record_custom_event("LlmChatCompletionSummary", error_chat_completion_dict) + + create_chat_completion_message_event( + transaction, + input_message_content, + input_role, + completion_id, + span_id, + trace_id, + # Passing the request model as the response model here since we do not have access to a response model + request_model, + llm_metadata, + # Pass an empty output_message_list since we didn't receive a response from the LLM + [], + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + +def _handle_generation_success(transaction, settings, linking_metadata, completion_id, kwargs, ft, return_val): + if not return_val: + return + + try: + response = return_val + # Response objects are pydantic models so this function call converts the response into a dict + if hasattr(response, "model_dump"): + response = response.model_dump() + _record_generation_success(transaction, settings, linking_metadata, completion_id, kwargs, ft, response) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + +def _record_generation_success(transaction, settings, linking_metadata, completion_id, kwargs, ft, response): + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + try: + # Parse response details + if response: + response_model = response.get("model_version") + # finish_reason is an enum, so grab just the stringified value from it to report + finish_reason = response.get("candidates")[0].get("finish_reason").value + output_message_list = [response.get("candidates")[0].get("content")] + token_usage = response.get("usage_metadata") or {} + else: + # Set all values to NoneTypes since we cannot access them through kwargs or another method that doesn't + # require the response object + response_model = None + output_message_list = [] + finish_reason = None + token_usage = {} + + # Parse request details + request_model = kwargs.get("model") + + # If generate_content was called directly, "contents" should hold a string with just the user input. + # If send_message was called, "contents" should hold a list containing the user input & role. + # When send_message is called multiple times within a chat conversation, "contents" will hold chat history with + # multiple lists to capture each input to the LLM (only inputs and not responses) + messages = kwargs.get("contents") + + if isinstance(messages, str): + input_message = messages + else: + try: + input_message = messages[-1] + except Exception: + input_message = None + _logger.warning( + "Unable to parse input message to Gemini LLM. Message content and role will be omitted from " + "corresponding LlmChatCompletionMessage event. " + ) + + input_message_content, input_role = _parse_input_message(input_message) if input_message else (None, None) + + # Parse output message content + # This list should have a length of 1 to represent the output message + # Parse the message text out to pass to any registered token counting callback + output_message_content = output_message_list[0].get("parts")[0].get("text") if output_message_list else None + + # Extract token counts from response object + if token_usage: + response_prompt_tokens = token_usage.get("prompt_token_count") + response_completion_tokens = token_usage.get("candidates_token_count") + response_total_tokens = token_usage.get("total_token_count") + else: + response_prompt_tokens = None + response_completion_tokens = None + response_total_tokens = None + + # Calculate token counts by checking if a callback is registered and if we have the necessary content to pass + # to it. If not, then we use the token counts provided in the response object + prompt_tokens = ( + settings.ai_monitoring.llm_token_count_callback(request_model, input_message_content) + if settings.ai_monitoring.llm_token_count_callback and input_message_content + else response_prompt_tokens + ) + completion_tokens = ( + settings.ai_monitoring.llm_token_count_callback(response_model, output_message_content) + if settings.ai_monitoring.llm_token_count_callback and output_message_content + else response_completion_tokens + ) + total_tokens = ( + prompt_tokens + completion_tokens if all([prompt_tokens, completion_tokens]) else response_total_tokens + ) + + # Extract generation config + request_temperature, request_max_tokens = _extract_generation_config(kwargs) + + full_chat_completion_summary_dict = { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "request.model": request_model, + "request.temperature": request_temperature, + "request.max_tokens": request_max_tokens, + "vendor": "gemini", + "ingest_source": "Python", + "duration": ft.duration * 1000, + "response.model": response_model, + "response.choices.finish_reason": finish_reason, + # Adding a 1 to the output_message_list length because we will only ever report the latest, single input + # message This value should be 2 in almost all cases since we will report a summary event for each + # separate request (every input and output from the LLM) + "response.number_of_messages": 1 + len(output_message_list), + "response.usage.prompt_tokens": prompt_tokens, + "response.usage.completion_tokens": completion_tokens, + "response.usage.total_tokens": total_tokens, + } + + llm_metadata = _get_llm_attributes(transaction) + full_chat_completion_summary_dict.update(llm_metadata) + transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) + + create_chat_completion_message_event( + transaction, + input_message_content, + input_role, + completion_id, + span_id, + trace_id, + response_model, + llm_metadata, + output_message_list, + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + +def _parse_input_message(input_message): + # The input_message will be a string if generate_content was called directly. In this case, we don't have + # access to the role, so we default to user since this was an input message + if isinstance(input_message, str): + return input_message, "user" + # The input_message will be a Google Content type if send_message was called, so we parse out the message + # text and role (which should be "user") + elif isinstance(input_message, google.genai.types.Content): + return input_message.parts[0].text, input_message.role + else: + return None, None + + +def _extract_generation_config(kwargs): + generation_config = kwargs.get("config") + if generation_config: + request_temperature = getattr(generation_config, "temperature", None) + request_max_tokens = getattr(generation_config, "max_output_tokens", None) + else: + request_temperature = None + request_max_tokens = None + + return request_temperature, request_max_tokens + + +def create_chat_completion_message_event( + transaction, + input_message_content, + input_role, + chat_completion_id, + span_id, + trace_id, + response_model, + llm_metadata, + output_message_list, +): + try: + settings = transaction.settings or global_settings() + + if input_message_content: + message_id = str(uuid.uuid4()) + + chat_completion_input_message_dict = { + "id": message_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": 0, + "role": input_role, + "completion_id": chat_completion_id, + # The input message will always be the first message in our request/ response sequence so this will + # always be 0 + "sequence": 0, + "response.model": response_model, + "vendor": "gemini", + "ingest_source": "Python", + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_input_message_dict["content"] = input_message_content + + chat_completion_input_message_dict.update(llm_metadata) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_input_message_dict) + + if output_message_list: + # Loop through all output messages received from the LLM response and emit a custom event for each one + # In almost all foreseeable cases, there should only be one item in this output_message_list + for index, message in enumerate(output_message_list): + message_content = message.get("parts")[0].get("text") + + # Add one to the index to account for the single input message so our sequence value is accurate for + # the output message + if input_message_content: + index += 1 + + message_id = str(uuid.uuid4()) + + chat_completion_output_message_dict = { + "id": message_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": 0, + "role": message.get("role"), + "completion_id": chat_completion_id, + "sequence": index, + "response.model": response_model, + "vendor": "gemini", + "ingest_source": "Python", + "is_response": True, + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_output_message_dict["content"] = message_content + + chat_completion_output_message_dict.update(llm_metadata) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_output_message_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + + +def instrument_genai_models(module): + if hasattr(module, "Models"): + wrap_function_wrapper(module, "Models.generate_content", wrap_generate_content_sync) + wrap_function_wrapper(module, "Models.embed_content", wrap_embed_content_sync) + + if hasattr(module, "AsyncModels"): + wrap_function_wrapper(module, "AsyncModels.generate_content", wrap_generate_content_async) + wrap_function_wrapper(module, "AsyncModels.embed_content", wrap_embed_content_async) diff --git a/tests/mlmodel_gemini/_mock_external_gemini_server.py b/tests/mlmodel_gemini/_mock_external_gemini_server.py new file mode 100644 index 0000000000..753bedb14d --- /dev/null +++ b/tests/mlmodel_gemini/_mock_external_gemini_server.py @@ -0,0 +1,979 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest +from testing_support.mock_external_http_server import MockExternalHTTPServer + +from newrelic.common.package_version_utils import get_package_version_tuple + +# This defines an external server test apps can make requests to instead of +# the real Google Gemini backend. This provides 3 features: +# +# 1) This removes dependencies on external websites. +# 2) Provides a better mechanism for making an external call in a test app than +# simple calling another endpoint the test app makes available because this +# server will not be instrumented meaning we don't have to sort through +# transactions to separate the ones created in the test app and the ones +# created by an external call. +# 3) This app runs on a separate thread meaning it won't block the test app. + +RESPONSES = { + "Invalid API key.": [ + { + "content-type": "application/json", + "x-goog-api-key": "GEMINI_API_KEY", + "x-goog-api-client": "google-genai-sdk/1.8.0 gl-python/3.11.4", + }, + 400, + { + "error": { + "code": 400, + "message": "API key not valid. Please pass a valid API key.", + "status": "INVALID_ARGUMENT", + "details": [ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "API_KEY_INVALID", + "domain": "googleapis.com", + "metadata": {"service": "generativelanguage.googleapis.com"}, + }, + { + "@type": "type.googleapis.com/google.rpc.LocalizedMessage", + "locale": "en-US", + "message": "API key not valid. Please pass a valid API key.", + }, + ], + } + }, + ], + "Model does not exist.": [ + { + "content-type": "application/json", + "x-goog-api-key": "GEMINI_API_KEY", + "x-goog-api-client": "google-genai-sdk/1.8.0 gl-python/3.11.4", + }, + 404, + { + "error": { + "code": 404, + "message": "models/does-not-exist is not found for API version v1beta, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods.", + "status": "NOT_FOUND", + } + }, + ], + "Embedded: Model does not exist.": [ + { + "content-type": "application/json", + "x-goog-api-key": "GEMINI_API_KEY", + "x-goog-api-client": "google-genai-sdk/1.8.0 gl-python/3.11.4", + }, + 404, + { + "error": { + "code": 404, + "message": "models/does-not-exist is not found for API version v1beta, or is not supported for embedContent. Call ListModels to see the list of available models and their supported methods.", + "status": "NOT_FOUND", + } + }, + ], + "How many letters are in the word Python?": [ + { + "content-type": "application/json", + "x-goog-api-key": "GEMINI_API_KEY", + "x-goog-api-client": "google-genai-sdk/1.8.0 gl-python/3.11.4", + }, + 200, + { + "candidates": [ + { + "content": { + "parts": [{"text": 'There are **6** letters in the word "Python".\n'}], + "role": "model", + }, + "finishReason": "STOP", + "avgLogprobs": -0.21492499571580154, + } + ], + "usageMetadata": { + "promptTokenCount": 9, + "candidatesTokenCount": 13, + "totalTokenCount": 22, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 13}], + }, + "modelVersion": "gemini-2.0-flash", + }, + ], + "This is an embedding test.": [ + { + "content-type": "application/json", + "x-goog-api-key": "GEMINI_API_KEY", + "x-goog-api-client": "google-genai-sdk/1.8.0 gl-python/3.11.4", + }, + 200, + { + "embedding": { + "values": [ + 0.0042326464, + 0.029004563, + -0.07853497, + 0.015845867, + 0.060363144, + 0.008264836, + 0.08827857, + 0.051790033, + -0.010739345, + 0.0097664865, + -0.0030682562, + 0.07532256, + 0.06018427, + -0.018718004, + 0.0030775273, + -0.06404092, + 0.01164792, + 0.010430611, + -0.06673716, + -0.011297725, + 0.026882315, + -0.0271604, + 0.0053434824, + -0.04982204, + -0.014138874, + -0.017344888, + 0.023262324, + -0.027566284, + 0.028769746, + -0.01790407, + 0.021286923, + 0.037220098, + 0.027963053, + 0.018125042, + -0.024682406, + -0.015903063, + 0.008201522, + 0.047491897, + 0.04206297, + -0.02044235, + -0.026820127, + 0.017688768, + -0.038202174, + 0.04803243, + -0.020497348, + -0.06809881, + 0.019102287, + -0.016282575, + -0.008147121, + 0.023472652, + 0.066234715, + 0.029089477, + -0.05138362, + -0.0050375434, + -0.025048234, + -0.027642297, + -0.0064551206, + -0.03406217, + -0.0003015566, + 0.01686922, + 0.051587597, + -0.05335174, + -0.015375454, + -0.006774749, + 0.017535774, + 0.0075883474, + -0.027056802, + -0.010513732, + -0.10870002, + 0.038311325, + -0.023443738, + 0.0006942114, + 0.0017571698, + 0.04022458, + 0.0053058867, + -0.013459642, + 0.037681907, + -0.031173995, + -0.019205242, + 0.046206266, + 0.0021427372, + 0.026726939, + 0.07782731, + 0.037151683, + 0.079537086, + -0.011208424, + 0.046385463, + -0.04847148, + -0.044214733, + -0.024537982, + 0.08815229, + -0.0019893285, + 0.02862184, + 0.04221302, + 0.051742513, + -0.040929493, + -0.025749704, + 0.015830496, + 0.023573006, + 0.107543774, + 0.020702802, + -0.007434051, + 0.013178751, + -0.014609835, + 0.0055954396, + 0.055712786, + -0.07954713, + 0.00041368845, + -0.075608104, + -0.007885142, + -0.010423114, + -0.018631916, + 0.027715273, + -0.04978383, + -0.050067883, + -0.079438716, + 0.014000716, + 0.030478386, + 0.009050721, + -0.025198627, + -0.0028137376, + 0.06723492, + -0.048240956, + 0.017409516, + 0.018344734, + 0.007830445, + -0.02456571, + -0.0030882207, + -0.0051813135, + -0.035331354, + 0.029751945, + -0.044173956, + 0.0015829265, + 0.012597751, + 0.030796584, + -0.03777858, + 0.072733246, + 0.06821187, + 0.024542531, + 0.06631259, + -0.03229986, + -0.04381717, + -0.106004834, + -0.0011497795, + -0.00930112, + -0.029783485, + 0.03572324, + 0.042525727, + -0.054525614, + -0.01470237, + 0.00053973304, + -0.0041476768, + -0.022199042, + 0.02284123, + -0.02174277, + 0.011732314, + 0.057626083, + -0.07894061, + 0.09304443, + -0.005694812, + 0.0051541748, + -0.049611423, + 0.037705578, + -0.025591472, + -0.021194516, + -0.027488453, + 0.049624503, + -0.070989944, + 0.01657545, + 0.013350953, + 0.00841555, + 0.013943526, + -0.032539055, + 0.0014774579, + -0.04796844, + -0.015522849, + -0.037159007, + -0.022781903, + 0.005494773, + 0.00034274854, + 0.03895854, + 0.040065814, + 0.031668853, + -0.10396242, + -0.060128905, + -0.006456444, + -0.0012389452, + 0.015883658, + 0.04947868, + 0.082508236, + -0.027179895, + 0.02871063, + 0.024578592, + -0.009762573, + 0.007507379, + 0.026203629, + 0.016447682, + -0.03294233, + -0.051270362, + 0.023154914, + 0.013433654, + -0.030371645, + 0.024437387, + -0.011202293, + -0.000749408, + 0.027105926, + -0.0388552, + -0.056164227, + 0.050209496, + 0.010824088, + -0.032063395, + -0.054537274, + 0.012184584, + -0.0067816237, + 0.018609297, + -0.0139637, + 0.02463699, + -0.0046339706, + 0.06117317, + -0.04965085, + -0.031187585, + 0.011864996, + -0.0012917097, + 0.060485847, + 0.035890013, + 0.0068307775, + 0.0043460564, + -0.0019570603, + -0.027023794, + -0.007598705, + 0.006394645, + 0.0058349767, + 0.042051516, + -0.023572199, + -0.033351462, + 0.031145293, + -0.0120965075, + -0.033105936, + -0.04605126, + -0.056910872, + -0.005600399, + 0.017727192, + -0.02330314, + 0.011734753, + 0.047223542, + 0.03492995, + 0.04730999, + 0.050469574, + 0.0036776129, + -0.020731065, + -0.05031616, + -0.040163178, + -0.08734381, + -0.052698817, + 0.00123459, + 0.010232824, + 0.025452584, + -0.0026405696, + -0.0037284044, + 0.018655455, + 0.06219949, + -0.017533502, + -0.0053410535, + -0.07754048, + -0.050167803, + -0.0760839, + -0.015050281, + 0.011657816, + 0.056118097, + -0.07960886, + 0.017400416, + -0.037782155, + -0.0134981265, + -0.014786434, + 0.01310445, + 0.067695916, + -0.002906271, + 0.012523397, + -0.06826953, + -0.021396834, + 0.026826175, + 0.040257465, + -0.007992264, + -0.046851274, + -0.052621055, + -0.059978846, + -0.0076254276, + -0.007630302, + -0.028449025, + -0.018142525, + 0.040817857, + 0.020701878, + 0.00365317, + -0.051722288, + 0.022785434, + -0.0126320375, + 0.0367689, + -0.0016271791, + 0.026618771, + -0.0023710267, + 0.014240093, + 0.055501103, + -0.031106712, + 0.013023728, + 0.029487276, + 0.018924234, + 0.005408001, + -0.07983805, + 0.0005971905, + 0.01940745, + 0.015026314, + 0.02359795, + 0.0054673213, + -0.07791104, + -0.048737913, + -0.064455554, + -0.12638979, + 0.0008822773, + -0.07627577, + -0.0061467024, + -0.0008589217, + -0.028389007, + -0.041662283, + -0.021177875, + 0.04416939, + -0.0036251508, + -0.013347197, + 0.020907475, + -0.023388114, + 0.005763718, + 0.022672728, + 0.010382343, + -0.0632251, + -0.04539014, + -0.0002913326, + -0.03358886, + -0.011878241, + 0.00057383516, + 0.045176484, + 0.0044332715, + 0.009115762, + -0.009695122, + 0.04399701, + 0.002461479, + 0.0063320496, + -0.038243365, + 0.014773573, + -0.041580588, + 0.025570251, + 0.00024998098, + 0.022766402, + 0.002696928, + 0.042013437, + 0.020654615, + -0.022486083, + 0.020701941, + 0.06769541, + -0.027665107, + 0.025065677, + -0.058677126, + -0.005606738, + -0.030660594, + 0.036190093, + 0.005217673, + 0.0038946, + -0.049907424, + 0.018475268, + 0.030534435, + -0.033634003, + -0.026514698, + 0.061727878, + 0.009071, + -0.020185351, + 0.002289446, + -0.018585907, + -0.0067302696, + -0.012673388, + 0.04209287, + 0.05002671, + -0.044933718, + -0.010536026, + -0.022743523, + -0.0019238136, + -0.0060753403, + -0.022139898, + 0.047112066, + -0.016026238, + 0.025906768, + 0.019289989, + 0.021619951, + -0.012293178, + -0.024648042, + 0.049239427, + 0.033746533, + 0.010492671, + -0.029039891, + -0.049868416, + 0.033278476, + -0.004040017, + 0.059626482, + -0.024160884, + -0.07498226, + 0.039521437, + 0.011464214, + -0.018192973, + -0.007372519, + 0.043203346, + 0.013612146, + -0.055734996, + -0.020606652, + 0.022268532, + 0.0025279694, + -0.0984952, + 0.032667574, + -0.06070344, + 0.029531656, + 0.020999247, + 0.01043111, + -0.03483265, + -0.004983468, + 0.009015729, + 0.013509287, + -0.016383434, + 0.050168797, + 0.043268766, + -0.07614634, + -0.02804231, + 0.0032657727, + 0.044405706, + -0.04485564, + 0.007100709, + 0.014508236, + -0.0069139157, + 0.004806404, + 0.017834296, + 0.0074579506, + -0.0012751953, + -0.020872502, + -0.030439794, + -0.023535244, + -0.004779637, + -0.012195425, + 0.02046227, + 0.01699531, + 0.0080058975, + 0.01996157, + -0.029094048, + -0.013285501, + -0.008434056, + -0.0021355576, + -0.0038001782, + -0.006549041, + -0.05960305, + -0.032245938, + -0.08058905, + 0.02066991, + 0.0017652615, + 0.002863885, + 0.004771491, + 0.03134235, + 0.0019840382, + 0.008926352, + 0.054349653, + 0.0126468595, + 0.047189254, + -0.045331728, + -0.0031931093, + 0.05145703, + 0.028904984, + 0.038406808, + 0.076466255, + -0.0082346415, + 0.041650582, + 0.00038202645, + -0.040065803, + -0.0034174302, + 0.07883703, + -0.024085721, + 0.015974889, + -0.014614687, + 0.008826982, + 0.013056139, + -0.036232814, + 0.0017404106, + 0.046567507, + -0.038627587, + -0.022772735, + -0.00360903, + 0.06401064, + 0.018614624, + 0.024335671, + -0.01810877, + 0.018877637, + 0.0076220147, + 0.027648995, + -0.02293826, + 0.02607716, + 0.0022417856, + 0.035924125, + 0.031712912, + 0.015599443, + 0.01184005, + -0.018949784, + -0.023008842, + 0.017906023, + 0.0064447913, + 0.0022186881, + -0.023603898, + 0.11304756, + 0.013100798, + 0.047045246, + -0.004973314, + 0.037342075, + -0.037097525, + 0.0064201783, + 0.016215287, + -0.04781635, + 0.04588995, + -0.04934734, + 0.032746654, + -0.030598218, + -0.023398431, + -0.0009910475, + -0.012254112, + -0.013276868, + -0.049342964, + 0.038995508, + -0.032407753, + 0.02308919, + -0.008438302, + -0.0784327, + 0.06612419, + 0.0011994164, + 0.04635904, + -0.018904384, + 0.06974694, + 0.01702173, + -0.01874133, + -0.029214252, + -0.020141654, + -0.0019626785, + 0.018563015, + 0.008967391, + 0.026343763, + 0.024692085, + 0.012153884, + -0.03099715, + 0.046586502, + 0.0030605833, + 0.037427466, + 0.022576723, + -0.01451993, + -0.0053356336, + 0.019817669, + -0.014174678, + -0.034082834, + -0.029313268, + 0.028502822, + -0.053297944, + 0.0057360632, + -0.009595574, + -0.012570278, + -0.049505513, + -0.06881855, + -0.019431435, + -0.020797614, + -0.010205221, + 0.0212843, + 0.0040956843, + -0.032163706, + -0.01579795, + 0.011154651, + 0.05165608, + -0.044125475, + 0.051611453, + 0.015573544, + -0.047152903, + -0.034506742, + 0.042775963, + -0.0063938797, + 0.022578692, + -0.000698978, + 0.11097389, + 0.0042039477, + -0.012744041, + 0.025101813, + 0.05587627, + -0.025829926, + -0.000377134, + 0.018274203, + -0.044377174, + 0.023601126, + -0.014993059, + -0.03146059, + -0.011369117, + -0.0068684905, + -0.047642265, + 0.023028033, + 0.05118852, + 0.037967626, + 0.03589492, + -0.034851607, + 0.05836855, + 0.007422612, + -0.0028261798, + -0.084631816, + -0.008416511, + -0.013297727, + 0.045844845, + 0.0017582007, + 0.039870862, + 0.038967937, + -0.049584012, + 0.013567134, + -0.00683099, + 0.01515794, + -0.020968834, + -0.046133906, + 0.04676231, + -0.0072222347, + -0.027787503, + -0.0074175396, + 0.021458842, + 0.0051179165, + -0.024867475, + -0.0020387261, + 0.0007908524, + 0.031947404, + 0.054386783, + 0.003670804, + -0.029436346, + 0.013517629, + 0.03133852, + -0.04301945, + -0.012450221, + -0.011596534, + 0.010312541, + -0.004814255, + 0.013594315, + -0.00032178903, + -0.008096347, + 0.010448234, + 0.018018147, + 0.012485202, + 0.00015640758, + 0.056029968, + -0.034684505, + -0.005945102, + 0.04540071, + 0.03652228, + 0.017559635, + -0.0029428585, + 0.031903584, + 0.003917594, + -0.064691976, + -0.039297286, + 0.010467662, + 0.0093484605, + -0.014891123, + -0.009175688, + -0.04573362, + 0.0035475963, + 0.022246609, + 0.03532225, + -0.017237678, + -0.034951076, + -0.023324898, + -0.027865818, + -0.010352668, + -0.029556517, + -0.01141666, + -0.023895623, + -0.008008368, + 0.034369104, + -0.035263885, + -0.0023880412, + 0.0037398383, + -0.043421715, + -0.026140437, + 0.017505916, + 0.01172007, + -0.0052012587, + -0.010666706, + -0.044013575, + -0.021390941, + 0.00925609, + 0.04468026, + -0.05654946, + 0.05758538, + 0.038788065, + -0.06254394, + 0.033485573, + -0.03277277, + 0.041688804, + -0.016151179, + 0.029675728, + -0.052232865, + 0.012067895, + -0.039755195, + -0.038339008, + -0.013288515, + 0.009655733, + -0.015840251, + -0.017354984, + -0.00071384717, + -0.043791853, + -0.009513481, + 0.023261022, + -0.04977938, + 0.01587062, + -0.019334927, + -0.0021081585, + 0.06377285, + 0.026044562, + -0.004681057, + 0.02466819, + 0.0100608235, + -0.027291354, + 0.019331526, + -0.017899454, + -0.036263637, + 0.0022282766, + 0.05268942, + 0.03951839, + -0.018724138, + -0.01991192, + -0.037556514, + 0.056540187, + -0.052383814, + 0.041265804, + 0.059506774, + 0.025743607, + -0.045918416, + 0.05303863, + -0.013928889, + -0.061523702, + 0.02737446, + 0.016407, + -0.03208383, + 0.047133777, + 0.020823127, + -0.049983423, + -0.094918594, + -0.039521795, + 0.021127349, + 0.0046334304, + -0.088479534, + -0.06331032, + -0.034863796, + -0.0043010265, + -0.09269081, + 0.019175887, + 0.012120077, + 0.02139774, + 0.038447183, + -0.06730395, + -0.0045507355, + -0.009664997, + 0.032972448, + -0.025034687, + -0.0015198331, + 0.082746305, + 0.0031048357, + -0.009439897, + -0.113036305, + -0.076955914, + 0.042393137, + -0.040755555, + ] + } + }, + ], +} + + +@pytest.fixture(scope="session") +def simple_get(extract_shortened_prompt): + def _simple_get(self): + content_len = int(self.headers.get("content-length")) + content = json.loads(self.rfile.read(content_len).decode("utf-8")) + + prompt = extract_shortened_prompt(content) + if not prompt: + self.send_response(500) + self.end_headers() + self.wfile.write("Could not parse prompt.".encode("utf-8")) + return + + headers, response = ({}, "") + + for k, v in RESPONSES.items(): + if prompt.startswith(k): + headers, status_code, response = v + break + else: # If no matches found + self.send_response(500) + self.end_headers() + self.wfile.write(("Unknown Prompt:\n%s" % prompt).encode("utf-8")) + return + + # Send response code + self.send_response(status_code) + + # Send headers + for k, v in headers.items(): + self.send_header(k, v) + self.end_headers() + + # Send response body + self.wfile.write(json.dumps(response).encode("utf-8")) + return + + return _simple_get + + +@pytest.fixture(scope="session") +def extract_shortened_prompt(): + def _extract_shortened_prompt(content): + try: + if "requests" in content.keys(): + prompt = content.get("requests")[0].get("content").get("parts")[0].get("text") + else: + prompt = content.get("contents")[0].get("parts")[0].get("text") + + prompt = prompt.lstrip().split("\n")[0] + except Exception: + prompt = "" + return prompt + + return _extract_shortened_prompt + + +@pytest.fixture(scope="session") +def MockExternalGeminiServer(simple_get): + class _MockExternalGeminiServer(MockExternalHTTPServer): + # To use this class in a test one needs to start and stop this server + # before and after making requests to the test app that makes the external + # calls. + + def __init__(self, handler=simple_get, port=None, *args, **kwargs): + super(_MockExternalGeminiServer, self).__init__(handler=handler, port=port, *args, **kwargs) # noqa: B026 + + return _MockExternalGeminiServer + + +if __name__ == "__main__": + with MockExternalGeminiServer() as server: + print("MockExternalGeminiServer serving on port %s" % str(server.port)) + while True: + pass # Serve forever diff --git a/tests/mlmodel_gemini/conftest.py b/tests/mlmodel_gemini/conftest.py new file mode 100644 index 0000000000..f30ae555e0 --- /dev/null +++ b/tests/mlmodel_gemini/conftest.py @@ -0,0 +1,138 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import google.genai +import pytest +from _mock_external_gemini_server import MockExternalGeminiServer, extract_shortened_prompt, simple_get +from testing_support.fixture.event_loop import event_loop as loop +from testing_support.fixtures import ( + collector_agent_registration_fixture, + collector_available_fixture, + override_application_settings, +) + +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.signature import bind_args + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow-downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "ml_insights_events.enabled": True, + "ai_monitoring.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (mlmodel_gemini)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (mlmodel_gemini)"], +) + + +GEMINI_AUDIT_LOG_FILE = os.path.join(os.path.realpath(os.path.dirname(__file__)), "gemini_audit.log") +GEMINI_AUDIT_LOG_CONTENTS = {} +# Intercept outgoing requests and log to file for mocking +RECORDED_HEADERS = set(["content-type"]) + + +@pytest.fixture(scope="session") +def gemini_clients(MockExternalGeminiServer): # noqa: F811 + """ + This configures the Gemini client and returns it + """ + from newrelic.core.config import _environ_as_bool + + if _environ_as_bool("NEW_RELIC_TESTING_RECORD_GEMINI_RESPONSES", True): + with MockExternalGeminiServer() as server: + gemini_dev_client = google.genai.Client( + api_key="GEMINI_API_KEY", + http_options=google.genai.types.HttpOptions(base_url=f"http://localhost:{server.port}"), + ) + yield gemini_dev_client + else: + google_api_key = os.environ.get("GOOGLE_API_KEY") + if not google_api_key: + raise RuntimeError("GOOGLE_API_KEY environment variable required.") + + gemini_dev_client = google.genai.Client(api_key=google_api_key) + yield gemini_dev_client + + +@pytest.fixture(scope="session") +def gemini_dev_client(gemini_clients): + # Once VertexAI is enabled, gemini_clients() will yield two different clients up that will be unpacked here + gemini_dev_client = gemini_clients + return gemini_dev_client + + +# @pytest.fixture(scope="session") +# def vertexai_client(gemini_clients): +# # This will eventually also be yielded up in gemini_clients() to test again the Vertex AI API +# _, vertexai_client = gemini_clients +# return vertexai_client + + +@pytest.fixture(autouse=True, scope="session") +def gemini_server(gemini_clients, wrap_httpx_client_send): + """ + This fixture will either create a mocked backend for testing purposes, or will + set up an audit log file to log responses of the real Gemini backend to a file. + The behavior can be controlled by setting NEW_RELIC_TESTING_RECORD_GEMINI_RESPONSES=1 as + an environment variable to run using the real Gemini backend. (Default: mocking) + """ + from newrelic.core.config import _environ_as_bool + + if _environ_as_bool("NEW_RELIC_TESTING_RECORD_GEMINI_RESPONSES", True): + wrap_function_wrapper("httpx._client", "Client.send", wrap_httpx_client_send) + yield # Run tests + # Write responses to audit log + with open(GEMINI_AUDIT_LOG_FILE, "w") as audit_log_fp: + json.dump(GEMINI_AUDIT_LOG_CONTENTS, fp=audit_log_fp, indent=4) + else: + # We are mocking responses so we don't need to do anything in this case. + yield + + +@pytest.fixture(scope="session") +def wrap_httpx_client_send(extract_shortened_prompt): # noqa: F811 + def _wrap_httpx_client_send(wrapped, instance, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + request = bound_args["request"] + if not request: + return wrapped(*args, **kwargs) + + params = json.loads(request.content.decode("utf-8")) + prompt = extract_shortened_prompt(params) + + # Send request + response = wrapped(*args, **kwargs) + + if response.status_code >= 500 or response.status_code < 200: + prompt = "error" + + rheaders = response.headers + headers = dict( + filter(lambda k: k[0].lower() in RECORDED_HEADERS or k[0].lower().startswith("x-goog"), rheaders.items()) + ) + body = json.loads(response.content.decode("utf-8")) + GEMINI_AUDIT_LOG_CONTENTS[prompt] = headers, response.status_code, body # Append response data to log + return response + + return _wrap_httpx_client_send diff --git a/tests/mlmodel_gemini/test_embeddings.py b/tests/mlmodel_gemini/test_embeddings.py new file mode 100644 index 0000000000..0fc92897b6 --- /dev/null +++ b/tests/mlmodel_gemini/test_embeddings.py @@ -0,0 +1,218 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.genai +from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + add_token_count_to_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_sans_content, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.transaction import add_custom_attribute + +embedding_recorded_events = [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "span_id": None, + "trace_id": "trace-id", + "input": "This is an embedding test.", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "duration": None, # Response time varies each test run + "request.model": "text-embedding-004", + "vendor": "gemini", + "ingest_source": "Python", + }, + ) +] + + +@reset_core_stats_engine() +@validate_custom_events(embedding_recorded_events) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + name="test_embeddings:test_gemini_embedding_sync", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_embedding_sync(gemini_dev_client, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + gemini_dev_client.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(events_sans_content(embedding_recorded_events)) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + name="test_embeddings:test_gemini_embedding_sync_no_content", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_embedding_sync_no_content(gemini_dev_client, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + gemini_dev_client.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_custom_events(add_token_count_to_events(embedding_recorded_events)) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + name="test_embeddings:test_gemini_embedding_sync_with_token_count", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_embedding_sync_with_token_count(gemini_dev_client, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + gemini_dev_client.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_gemini_embedding_sync_outside_txn(gemini_dev_client): + gemini_dev_client.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_gemini_embedding_sync_disabled_ai_monitoring_events(gemini_dev_client, set_trace_info): + set_trace_info() + gemini_dev_client.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + + +@reset_core_stats_engine() +@validate_custom_events(embedding_recorded_events) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + name="test_embeddings:test_gemini_embedding_async", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_embedding_async(gemini_dev_client, loop, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + loop.run_until_complete( + gemini_dev_client.aio.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(events_sans_content(embedding_recorded_events)) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + name="test_embeddings:test_gemini_embedding_async_no_content", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_embedding_async_no_content(gemini_dev_client, loop, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + loop.run_until_complete( + gemini_dev_client.aio.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + ) + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_custom_events(add_token_count_to_events(embedding_recorded_events)) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + name="test_embeddings:test_gemini_embedding_async_with_token_count", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_embedding_async_with_token_count(gemini_dev_client, loop, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + loop.run_until_complete( + gemini_dev_client.aio.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + ) + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_gemini_embedding_async_outside_txn(gemini_dev_client, loop): + loop.run_until_complete( + gemini_dev_client.aio.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + ) + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_gemini_embedding_async_disabled_ai_monitoring_events(gemini_dev_client, loop, set_trace_info): + set_trace_info() + loop.run_until_complete( + gemini_dev_client.aio.models.embed_content(contents="This is an embedding test.", model="text-embedding-004") + ) diff --git a/tests/mlmodel_gemini/test_embeddings_error.py b/tests/mlmodel_gemini/test_embeddings_error.py new file mode 100644 index 0000000000..a65a6c2c6f --- /dev/null +++ b/tests/mlmodel_gemini/test_embeddings_error.py @@ -0,0 +1,383 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import google.genai +import pytest +from testing_support.fixtures import dt_enabled, override_llm_token_callback_settings, reset_core_stats_engine +from testing_support.ml_testing_utils import ( + add_token_count_to_events, + disabled_ai_monitoring_record_content_settings, + events_sans_content, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.common.object_names import callable_name + +embedding_recorded_events = [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "span_id": None, + "trace_id": "trace-id", + "input": "This is an embedding test.", + "duration": None, # Response time varies each test run + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ) +] + + +def test_embeddings_invalid_request_error_no_model(gemini_dev_client, set_trace_info): + if sys.version_info < (3, 10): + error_message = "embed_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.embed_content() missing 1 required keyword-only argument: 'model'" + + # No model provided + @dt_enabled + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_invalid_request_error_no_model.._test", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, + ) + @validate_custom_events(embedding_recorded_events) + @validate_custom_event_count(count=1) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + gemini_dev_client.models.embed_content( + contents="This is an embedding test." + # No model + ) + + _test() + + +def test_embeddings_invalid_request_error_no_model_no_content(gemini_dev_client, set_trace_info): + if sys.version_info < (3, 10): + error_message = "embed_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.embed_content() missing 1 required keyword-only argument: 'model'" + + @dt_enabled + @disabled_ai_monitoring_record_content_settings + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_invalid_request_error_no_model_no_content.._test", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, + ) + @validate_custom_events(events_sans_content(embedding_recorded_events)) + @validate_custom_event_count(count=1) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + gemini_dev_client.models.embed_content( + contents="This is an embedding test." + # no model provided + ) + + _test() + + +invalid_model_events = [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "span_id": None, + "trace_id": "trace-id", + "input": "Embedded: Model does not exist.", + "duration": None, # Response time varies each test run + "request.model": "does-not-exist", # No model in this test case + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ) +] + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for embedContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_invalid_request_error_invalid_model", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_custom_events(invalid_model_events) +@validate_custom_event_count(count=1) +@background_task() +def test_embeddings_invalid_request_error_invalid_model(gemini_dev_client, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + gemini_dev_client.models.embed_content(contents="Embedded: Model does not exist.", model="does-not-exist") + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for embedContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_invalid_request_error_invalid_model_with_token_count", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_custom_events(add_token_count_to_events(invalid_model_events)) +@validate_custom_event_count(count=1) +@background_task() +def test_embeddings_invalid_request_error_invalid_model_with_token_count(gemini_dev_client, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + gemini_dev_client.models.embed_content(contents="Embedded: Model does not exist.", model="does-not-exist") + + +embedding_invalid_key_error_events = [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "span_id": None, + "trace_id": "trace-id", + "input": "Invalid API key.", + "duration": None, # Response time varies each test run + "request.model": "text-embedding-004", # No model in this test case + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ) +] + + +# Wrong api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "INVALID_ARGUMENT", "http.statusCode": 400}}, +) +@validate_span_events(exact_agents={"error.message": "API key not valid. Please pass a valid API key."}) +@validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_wrong_api_key_error", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_custom_events(embedding_invalid_key_error_events) +@validate_custom_event_count(count=1) +@background_task() +def test_embeddings_wrong_api_key_error(monkeypatch, gemini_dev_client, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + gemini_dev_client._api_client.api_key = "DEADBEEF" + gemini_dev_client.models.embed_content(contents="Invalid API key.", model="text-embedding-004") + + +def test_embeddings_async_invalid_request_error_no_model(gemini_dev_client, loop, set_trace_info): + if sys.version_info < (3, 10): + error_message = "embed_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.embed_content() missing 1 required keyword-only argument: 'model'" + + @dt_enabled + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_async_invalid_request_error_no_model.._test", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, + ) + @validate_custom_events(embedding_recorded_events) + @validate_custom_event_count(count=1) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + loop.run_until_complete( + gemini_dev_client.models.embed_content( + contents="This is an embedding test." + # No model + ) + ) + + _test() + + +def test_embeddings_async_invalid_request_error_no_model_no_content(gemini_dev_client, loop, set_trace_info): + if sys.version_info < (3, 10): + error_message = "embed_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.embed_content() missing 1 required keyword-only argument: 'model'" + + @dt_enabled + @disabled_ai_monitoring_record_content_settings + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_async_invalid_request_error_no_model_no_content.._test", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, + ) + @validate_custom_events(events_sans_content(embedding_recorded_events)) + @validate_custom_event_count(count=1) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + loop.run_until_complete( + gemini_dev_client.models.embed_content( + contents="This is an embedding test." + # no model provided + ) + ) + + _test() + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for embedContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_async_invalid_request_error_invalid_model", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_custom_events(invalid_model_events) +@validate_custom_event_count(count=1) +@background_task() +def test_embeddings_async_invalid_request_error_invalid_model(gemini_dev_client, loop, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + loop.run_until_complete( + gemini_dev_client.models.embed_content(contents="Embedded: Model does not exist.", model="does-not-exist") + ) + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for embedContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_async_invalid_request_error_invalid_model_with_token_count", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_custom_events(add_token_count_to_events(invalid_model_events)) +@validate_custom_event_count(count=1) +@background_task() +def test_embeddings_async_invalid_request_error_invalid_model_with_token_count(gemini_dev_client, loop, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + loop.run_until_complete( + gemini_dev_client.models.embed_content(contents="Embedded: Model does not exist.", model="does-not-exist") + ) + + +# Wrong api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "INVALID_ARGUMENT", "http.statusCode": 400}}, +) +@validate_span_events(exact_agents={"error.message": "API key not valid. Please pass a valid API key."}) +@validate_transaction_metrics( + name="test_embeddings_error:test_embeddings_async_wrong_api_key_error", + scoped_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + rollup_metrics=[("Llm/embedding/Gemini/embed_content", 1)], + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_custom_events(embedding_invalid_key_error_events) +@validate_custom_event_count(count=1) +@background_task() +def test_embeddings_async_wrong_api_key_error(gemini_dev_client, loop, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + gemini_dev_client._api_client.api_key = "DEADBEEF" + loop.run_until_complete( + gemini_dev_client.models.embed_content(contents="Invalid API key.", model="text-embedding-004") + ) diff --git a/tests/mlmodel_gemini/test_text_generation.py b/tests/mlmodel_gemini/test_text_generation.py new file mode 100644 index 0000000000..a708393019 --- /dev/null +++ b/tests/mlmodel_gemini/test_text_generation.py @@ -0,0 +1,401 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.genai +from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + add_token_counts_to_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_sans_content, + events_sans_llm_metadata, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute + +text_generation_recorded_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "gemini-2.0-flash", + "response.model": "gemini-2.0-flash", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "STOP", + "vendor": "gemini", + "ingest_source": "Python", + "response.number_of_messages": 2, + "response.usage.prompt_tokens": 9, + "response.usage.completion_tokens": 13, + "response.usage.total_tokens": 22, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "token_count": 0, + "content": "How many letters are in the word Python?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "gemini-2.0-flash", + "vendor": "gemini", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "token_count": 0, + "content": 'There are **6** letters in the word "Python".\n', + "role": "model", + "completion_id": None, + "sequence": 1, + "response.model": "gemini-2.0-flash", + "vendor": "gemini", + "is_response": True, + "ingest_source": "Python", + }, + ), +] + + +@reset_core_stats_engine() +# Expect one summary event, one message event for the input, and message event for the output for each send_message_call +@validate_custom_event_count(count=6) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_multi_text_generation", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_multi_text_generation(gemini_dev_client, set_trace_info): + chat = gemini_dev_client.chats.create(model="gemini-2.0-flash") + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + chat.send_message( + message="Who invented the Python programming language?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_sync_generate_content", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_sync_generate_content(gemini_dev_client, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + gemini_dev_client.models.generate_content( + model="gemini-2.0-flash", + contents="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_sync_with_llm_metadata", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_sync_with_llm_metadata(gemini_dev_client, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + chat = gemini_dev_client.chats.create(model="gemini-2.0-flash") + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(events_sans_content(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_sync_no_content", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_sync_no_content(gemini_dev_client, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + + chat = gemini_dev_client.chats.create(model="gemini-2.0-flash") + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_custom_events(add_token_counts_to_events(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_sync_with_token_count", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_sync_with_token_count(gemini_dev_client, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + + chat = gemini_dev_client.chats.create(model="gemini-2.0-flash") + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@reset_core_stats_engine() +@validate_custom_events(events_sans_llm_metadata(text_generation_recorded_events)) +# One summary event, one system message, one user message, and one response message from the assistant +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + "test_text_generation:test_gemini_text_generation_sync_no_llm_metadata", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@background_task() +def test_gemini_text_generation_sync_no_llm_metadata(gemini_dev_client, set_trace_info): + set_trace_info() + + chat = gemini_dev_client.chats.create(model="gemini-2.0-flash") + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_gemini_text_generation_sync_outside_txn(gemini_dev_client): + chat = gemini_dev_client.chats.create(model="gemini-2.0-flash") + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_gemini_text_generation_sync_ai_monitoring_disabled(gemini_dev_client): + chat = gemini_dev_client.chats.create(model="gemini-2.0-flash") + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_async_generate_content", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_async_generate_content(gemini_dev_client, loop, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete( + gemini_dev_client.aio.models.generate_content( + model="gemini-2.0-flash", + contents="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_async_with_llm_metadata", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_async_with_llm_metadata(gemini_dev_client, loop, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + chat = gemini_dev_client.aio.chats.create(model="gemini-2.0-flash") + loop.run_until_complete( + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(events_sans_content(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_async_no_content", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_async_no_content(gemini_dev_client, loop, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + + chat = gemini_dev_client.aio.chats.create(model="gemini-2.0-flash") + loop.run_until_complete( + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_custom_events(add_token_counts_to_events(text_generation_recorded_events)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_text_generation:test_gemini_text_generation_async_with_token_count", + custom_metrics=[(f"Supportability/Python/ML/Gemini/{google.genai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_text_generation_async_with_token_count(gemini_dev_client, loop, set_trace_info): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + + chat = gemini_dev_client.aio.chats.create(model="gemini-2.0-flash") + loop.run_until_complete( + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +@reset_core_stats_engine() +@validate_custom_events(events_sans_llm_metadata(text_generation_recorded_events)) +# One summary event, one system message, one user message, and one response message from the assistant +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + "test_text_generation:test_gemini_text_generation_async_no_llm_metadata", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@background_task() +def test_gemini_text_generation_async_no_llm_metadata(gemini_dev_client, loop, set_trace_info): + set_trace_info() + + chat = gemini_dev_client.aio.chats.create(model="gemini-2.0-flash") + loop.run_until_complete( + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_gemini_text_generation_async_outside_txn(gemini_dev_client, loop): + chat = gemini_dev_client.aio.chats.create(model="gemini-2.0-flash") + loop.run_until_complete( + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_gemini_text_generation_async_ai_monitoring_disabled(gemini_dev_client, loop): + chat = gemini_dev_client.aio.chats.create(model="gemini-2.0-flash") + loop.run_until_complete( + chat.send_message( + message="How many letters are in the word Python?", + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) diff --git a/tests/mlmodel_gemini/test_text_generation_error.py b/tests/mlmodel_gemini/test_text_generation_error.py new file mode 100644 index 0000000000..1460430b85 --- /dev/null +++ b/tests/mlmodel_gemini/test_text_generation_error.py @@ -0,0 +1,478 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys + +import google.genai +import pytest +from testing_support.fixtures import dt_enabled, override_llm_token_callback_settings, reset_core_stats_engine +from testing_support.ml_testing_utils import ( + add_token_counts_to_events_error, + disabled_ai_monitoring_record_content_settings, + events_sans_content, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name + +expected_events_on_no_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "token_count": 0, + "content": "How many letters are in the word Python?", + "role": "user", + "completion_id": None, + "sequence": 0, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), +] + + +def test_text_generation_invalid_request_error_no_model(gemini_dev_client, set_trace_info): + if sys.version_info < (3, 10): + error_message = "generate_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.generate_content() missing 1 required keyword-only argument: 'model'" + + # No model provided + @dt_enabled + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + "test_text_generation_error:test_text_generation_invalid_request_error_no_model.._test", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, + ) + @validate_custom_events(events_with_context_attrs(expected_events_on_no_model_error)) + @validate_custom_event_count(count=2) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + gemini_dev_client.models.generate_content( + # no model + contents=["How many letters are in the word Python?"], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + _test() + + +def test_text_generation_invalid_request_error_no_model_no_content(gemini_dev_client, set_trace_info): + if sys.version_info < (3, 10): + error_message = "generate_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.generate_content() missing 1 required keyword-only argument: 'model'" + + @dt_enabled + @disabled_ai_monitoring_record_content_settings + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + "test_text_generation_error:test_text_generation_invalid_request_error_no_model_no_content.._test", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, + ) + @validate_custom_events(events_sans_content(expected_events_on_no_model_error)) + @validate_custom_event_count(count=2) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + + gemini_dev_client.models.generate_content( + # no model + contents=["How many letters are in the word Python?"], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + _test() + + +expected_events_on_invalid_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "does-not-exist", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "token_count": 0, + "content": "Model does not exist.", + "role": "user", + "completion_id": None, + "response.model": "does-not-exist", + "sequence": 0, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), +] + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + "test_text_generation_error:test_text_generation_invalid_request_error_invalid_model_with_token_count", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@validate_custom_events(add_token_counts_to_events_error(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=2) +@background_task() +def test_text_generation_invalid_request_error_invalid_model_with_token_count(gemini_dev_client, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + gemini_dev_client.models.generate_content( + model="does-not-exist", + contents=["Model does not exist."], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + "test_text_generation_error:test_text_generation_invalid_request_error_invalid_model_chat", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@validate_custom_events(add_token_counts_to_events_error(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=2) +@background_task() +def test_text_generation_invalid_request_error_invalid_model_chat(gemini_dev_client, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + chat = gemini_dev_client.chats.create(model="does-not-exist") + chat.send_message( + message=["Model does not exist."], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +expected_events_on_wrong_api_key_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "gemini-flash-2.0", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "span_id": None, + "trace_id": "trace-id", + "token_count": 0, + "content": "Invalid API key.", + "role": "user", + "response.model": "gemini-flash-2.0", + "completion_id": None, + "sequence": 0, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), +] + + +# Wrong api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "INVALID_ARGUMENT", "http.statusCode": 400}}, +) +@validate_span_events(exact_agents={"error.message": "API key not valid. Please pass a valid API key."}) +@validate_transaction_metrics( + "test_text_generation_error:test_text_generation_wrong_api_key_error", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_wrong_api_key_error) +@validate_custom_event_count(count=2) +@background_task() +def test_text_generation_wrong_api_key_error(gemini_dev_client, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + gemini_dev_client._api_client.api_key = "DEADBEEF" + gemini_dev_client.models.generate_content( + model="gemini-flash-2.0", + contents=["Invalid API key."], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + + +def test_text_generation_async_invalid_request_error_no_model(gemini_dev_client, loop, set_trace_info): + if sys.version_info < (3, 10): + error_message = "generate_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.generate_content() missing 1 required keyword-only argument: 'model'" + + # No model provided + @dt_enabled + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + "test_text_generation_error:test_text_generation_async_invalid_request_error_no_model.._test", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, + ) + @validate_custom_events(events_with_context_attrs(expected_events_on_no_model_error)) + @validate_custom_event_count(count=2) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete( + gemini_dev_client.models.generate_content( + # no model + contents=["How many letters are in the word Python?"], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + _test() + + +def test_text_generation_async_invalid_request_error_no_model_no_content(gemini_dev_client, loop, set_trace_info): + if sys.version_info < (3, 10): + error_message = "generate_content() missing 1 required keyword-only argument: 'model'" + else: + error_message = "Models.generate_content() missing 1 required keyword-only argument: 'model'" + + @dt_enabled + @disabled_ai_monitoring_record_content_settings + @reset_core_stats_engine() + @validate_error_trace_attributes(callable_name(TypeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) + @validate_span_events(exact_agents={"error.message": error_message}) + @validate_transaction_metrics( + "test_text_generation_error:test_text_generation_async_invalid_request_error_no_model_no_content.._test", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, + ) + @validate_custom_events(events_sans_content(expected_events_on_no_model_error)) + @validate_custom_event_count(count=2) + @background_task() + def _test(): + with pytest.raises(TypeError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + loop.run_until_complete( + gemini_dev_client.models.generate_content( + # no model + contents=["How many letters are in the word Python?"], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + _test() + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + "test_text_generation_error:test_text_generation_async_invalid_request_error_invalid_model_with_token_count", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@validate_custom_events(add_token_counts_to_events_error(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=2) +@background_task() +def test_text_generation_async_invalid_request_error_invalid_model_with_token_count( + gemini_dev_client, loop, set_trace_info +): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + loop.run_until_complete( + gemini_dev_client.models.generate_content( + model="does-not-exist", + contents=["Model does not exist."], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "NOT_FOUND", "http.statusCode": 404}}, +) +@validate_span_events( + exact_agents={ + "error.message": "models/does-not-exist is not found for API version v1beta, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods." + } +) +@validate_transaction_metrics( + "test_text_generation_error:test_text_generation_async_invalid_request_error_invalid_model_chat", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@validate_custom_events(add_token_counts_to_events_error(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=2) +@background_task() +def test_text_generation_async_invalid_request_error_invalid_model_chat(gemini_dev_client, loop, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + chat = gemini_dev_client.chats.create(model="does-not-exist") + loop.run_until_complete( + chat.send_message( + message=["Model does not exist."], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) + + +# Wrong api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(google.genai.errors.ClientError), + exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "INVALID_ARGUMENT", "http.statusCode": 400}}, +) +@validate_span_events(exact_agents={"error.message": "API key not valid. Please pass a valid API key."}) +@validate_transaction_metrics( + "test_text_generation_error:test_text_generation_async_wrong_api_key_error", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_wrong_api_key_error) +@validate_custom_event_count(count=2) +@background_task() +def test_text_generation_async_wrong_api_key_error(gemini_dev_client, loop, set_trace_info): + with pytest.raises(google.genai.errors.ClientError): + set_trace_info() + gemini_dev_client._api_client.api_key = "DEADBEEF" + loop.run_until_complete( + gemini_dev_client.models.generate_content( + model="gemini-flash-2.0", + contents=["Invalid API key."], + config=google.genai.types.GenerateContentConfig(max_output_tokens=100, temperature=0.7), + ) + ) diff --git a/tests/testing_support/ml_testing_utils.py b/tests/testing_support/ml_testing_utils.py index 0e7307bfb0..2450b5333b 100644 --- a/tests/testing_support/ml_testing_utils.py +++ b/tests/testing_support/ml_testing_utils.py @@ -37,6 +37,25 @@ def add_token_count_to_events(expected_events): return events +def add_token_counts_to_events(expected_events): + events = copy.deepcopy(expected_events) + for event in events: + if event[0]["type"] == "LlmChatCompletionSummary": + event[1]["response.usage.prompt_tokens"] = 105 + event[1]["response.usage.completion_tokens"] = 105 + event[1]["response.usage.total_tokens"] = 210 + return events + + +def add_token_counts_to_events_error(expected_events): + events = copy.deepcopy(expected_events) + for event in events: + if event[0]["type"] == "LlmChatCompletionSummary": + event[1]["response.usage.prompt_tokens"] = 105 + event[1]["response.usage.total_tokens"] = 105 + return events + + def events_sans_content(event): new_event = copy.deepcopy(event) for _event in new_event: diff --git a/tox.ini b/tox.ini index a55832a4d3..66cfc7602b 100644 --- a/tox.ini +++ b/tox.ini @@ -152,6 +152,7 @@ envlist = python-logger_logging-{py37,py38,py39,py310,py311,py312,py313,pypy310}, python-logger_loguru-{py37,py38,py39,py310,py311,py312,py313,pypy310}-logurulatest, python-logger_structlog-{py37,py38,py39,py310,py311,py312,py313,pypy310}-structloglatest, + python-mlmodel_gemini-{py39,py310,py311,py312,py313}, python-mlmodel_langchain-{py39,py310,py311,py312}, ;; Package not ready for Python 3.13 (uses an older version of numpy) ; python-mlmodel_langchain-py313, @@ -392,6 +393,7 @@ deps = framework_tornado: pycurl framework_tornado-tornadolatest: tornado framework_tornado-tornadomaster: https://github.com/tornadoweb/tornado/archive/master.zip + mlmodel_gemini: google-genai mlmodel_openai-openai0: openai[datalib]<1.0 mlmodel_openai-openai107: openai[datalib]<1.8 mlmodel_openai-openai107: httpx<0.28 @@ -541,6 +543,7 @@ changedir = messagebroker_kafkapython: tests/messagebroker_kafkapython messagebroker_kombu: tests/messagebroker_kombu messagebroker_pika: tests/messagebroker_pika + mlmodel_gemini: tests/mlmodel_gemini mlmodel_langchain: tests/mlmodel_langchain mlmodel_openai: tests/mlmodel_openai mlmodel_sklearn: tests/mlmodel_sklearn