diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dc0a8ad3a..b531b99119 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- `opentelemetry-instrumentation-botocore` Add support for GenAI user events and lazy initialize tracer + ([#3258](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3258)) + ### Fixed + - `opentelemetry-instrumentation-redis` Add missing entry in doc string for `def _instrument` ([#3247](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3247)) diff --git a/instrumentation/opentelemetry-instrumentation-botocore/examples/bedrock-runtime/zero-code/requirements.txt b/instrumentation/opentelemetry-instrumentation-botocore/examples/bedrock-runtime/zero-code/requirements.txt index dea6c40109..93967de55c 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/examples/bedrock-runtime/zero-code/requirements.txt +++ b/instrumentation/opentelemetry-instrumentation-botocore/examples/bedrock-runtime/zero-code/requirements.txt @@ -1,6 +1,6 @@ boto3~=1.35.99 -opentelemetry-sdk~=1.29.0 -opentelemetry-exporter-otlp-proto-grpc~=1.29.0 -opentelemetry-distro~=0.50b0 -opentelemetry-instrumentation-botocore~=0.50b0 +opentelemetry-sdk~=1.30.0 +opentelemetry-exporter-otlp-proto-grpc~=1.30.0 +opentelemetry-distro~=0.51b0 +opentelemetry-instrumentation-botocore~=0.51b0 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/__init__.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/__init__.py index b5598a3cf7..39d339ae0c 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/__init__.py @@ -86,9 +86,15 @@ def response_hook(span, service_name, operation_name, result): from botocore.exceptions import ClientError from wrapt import wrap_function_wrapper -from opentelemetry.instrumentation.botocore.extensions import _find_extension +from opentelemetry._events import get_event_logger +from opentelemetry.instrumentation.botocore.extensions import ( + _find_extension, + _has_extension, +) from opentelemetry.instrumentation.botocore.extensions.types import ( _AwsSdkCallContext, + _AwsSdkExtension, + _BotocoreInstrumentorContext, ) from opentelemetry.instrumentation.botocore.package import _instruments from opentelemetry.instrumentation.botocore.version import __version__ @@ -123,12 +129,11 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs): # pylint: disable=attribute-defined-outside-init - self._tracer = get_tracer( - __name__, - __version__, - kwargs.get("tracer_provider"), - schema_url="https://opentelemetry.io/schemas/1.11.0", - ) + + # tracers are lazy initialized per-extension in _get_tracer + self._tracers = {} + # event_loggers are lazy initialized per-extension in _get_event_logger + self._event_loggers = {} self.request_hook = kwargs.get("request_hook") self.response_hook = kwargs.get("response_hook") @@ -137,6 +142,9 @@ def _instrument(self, **kwargs): if propagator is not None: self.propagator = propagator + self.tracer_provider = kwargs.get("tracer_provider") + self.event_logger_provider = kwargs.get("event_logger_provider") + wrap_function_wrapper( "botocore.client", "BaseClient._make_api_call", @@ -149,6 +157,50 @@ def _instrument(self, **kwargs): self._patched_endpoint_prepare_request, ) + @staticmethod + def _get_instrumentation_name(extension: _AwsSdkExtension) -> str: + has_extension = _has_extension(extension._call_context) + return ( + f"{__name__}.{extension._call_context.service}" + if has_extension + else __name__ + ) + + def _get_tracer(self, extension: _AwsSdkExtension): + """This is a multiplexer in order to have a tracer per extension""" + + instrumentation_name = self._get_instrumentation_name(extension) + tracer = self._tracers.get(instrumentation_name) + if tracer: + return tracer + + schema_version = extension.tracer_schema_version() + self._tracers[instrumentation_name] = get_tracer( + instrumentation_name, + __version__, + self.tracer_provider, + schema_url=f"https://opentelemetry.io/schemas/{schema_version}", + ) + return self._tracers[instrumentation_name] + + def _get_event_logger(self, extension: _AwsSdkExtension): + """This is a multiplexer in order to have an event logger per extension""" + + instrumentation_name = self._get_instrumentation_name(extension) + event_logger = self._event_loggers.get(instrumentation_name) + if event_logger: + return event_logger + + schema_version = extension.event_logger_schema_version() + self._event_loggers[instrumentation_name] = get_event_logger( + instrumentation_name, + "", + schema_url=f"https://opentelemetry.io/schemas/{schema_version}", + event_logger_provider=self.event_logger_provider, + ) + + return self._event_loggers[instrumentation_name] + def _uninstrument(self, **kwargs): unwrap(BaseClient, "_make_api_call") unwrap(Endpoint, "prepare_request") @@ -190,7 +242,12 @@ def _patched_api_call(self, original_func, instance, args, kwargs): _safe_invoke(extension.extract_attributes, attributes) end_span_on_exit = extension.should_end_span_on_exit() - with self._tracer.start_as_current_span( + tracer = self._get_tracer(extension) + event_logger = self._get_event_logger(extension) + instrumentor_ctx = _BotocoreInstrumentorContext( + event_logger=event_logger + ) + with tracer.start_as_current_span( call_context.span_name, kind=call_context.span_kind, attributes=attributes, @@ -198,7 +255,7 @@ def _patched_api_call(self, original_func, instance, args, kwargs): # at a later time after the stream has been consumed end_on_exit=end_span_on_exit, ) as span: - _safe_invoke(extension.before_service_call, span) + _safe_invoke(extension.before_service_call, span, instrumentor_ctx) self._call_request_hook(span, call_context) try: @@ -209,12 +266,16 @@ def _patched_api_call(self, original_func, instance, args, kwargs): except ClientError as error: result = getattr(error, "response", None) _apply_response_attributes(span, result) - _safe_invoke(extension.on_error, span, error) + _safe_invoke( + extension.on_error, span, error, instrumentor_ctx + ) raise _apply_response_attributes(span, result) - _safe_invoke(extension.on_success, span, result) + _safe_invoke( + extension.on_success, span, result, instrumentor_ctx + ) finally: - _safe_invoke(extension.after_service_call) + _safe_invoke(extension.after_service_call, instrumentor_ctx) self._call_response_hook(span, call_context, result) return result diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/__init__.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/__init__.py index c4624ababd..c1be7ec96f 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/__init__.py @@ -40,6 +40,10 @@ def loader(): } +def _has_extension(call_context: _AwsSdkCallContext) -> bool: + return call_context.service in _KNOWN_EXTENSIONS + + def _find_extension(call_context: _AwsSdkCallContext) -> _AwsSdkExtension: try: loader = _KNOWN_EXTENSIONS.get(call_context.service) diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py index 6d6bbce6ac..346e72772a 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py @@ -29,11 +29,14 @@ from opentelemetry.instrumentation.botocore.extensions.bedrock_utils import ( ConverseStreamWrapper, InvokeModelWithResponseStreamWrapper, + genai_capture_message_content, + message_to_event, ) from opentelemetry.instrumentation.botocore.extensions.types import ( _AttributeMapT, _AwsSdkExtension, _BotoClientErrorT, + _BotocoreInstrumentorContext, ) from opentelemetry.semconv._incubating.attributes.error_attributes import ( ERROR_TYPE, @@ -205,10 +208,34 @@ def _set_if_not_none(attributes, key, value): if value is not None: attributes[key] = value - def before_service_call(self, span: Span): + def _get_request_messages(self): + input_text = None + if not (messages := self._call_context.params.get("messages", [])): + if body := self._call_context.params.get("body"): + decoded_body = json.loads(body) + messages = decoded_body.get("messages", []) + if not messages: + # transform old school amazon titan invokeModel api to messages + if input_text := decoded_body.get("inputText"): + messages = [ + {"role": "user", "content": [{"text": input_text}]} + ] + + return messages + + def before_service_call( + self, span: Span, instrumentor_context: _BotocoreInstrumentorContext + ): if self._call_context.operation not in self._HANDLED_OPERATIONS: return + _capture_content = genai_capture_message_content() + + messages = self._get_request_messages() + for message in messages: + event_logger = instrumentor_context.event_logger + event_logger.emit(message_to_event(message, _capture_content)) + if not span.is_recording(): return @@ -272,7 +299,12 @@ def _on_stream_error_callback(self, span: Span, exception): span.set_attribute(ERROR_TYPE, type(exception).__qualname__) span.end() - def on_success(self, span: Span, result: dict[str, Any]): + def on_success( + self, + span: Span, + result: dict[str, Any], + instrumentor_context: _BotocoreInstrumentorContext, + ): if self._call_context.operation not in self._HANDLED_OPERATIONS: return @@ -384,7 +416,12 @@ def _handle_anthropic_claude_response( GEN_AI_RESPONSE_FINISH_REASONS, [response_body["stop_reason"]] ) - def on_error(self, span: Span, exception: _BotoClientErrorT): + def on_error( + self, + span: Span, + exception: _BotoClientErrorT, + instrumentor_context: _BotocoreInstrumentorContext, + ): if self._call_context.operation not in self._HANDLED_OPERATIONS: return diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock_utils.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock_utils.py index 8d0d806f43..28579c993a 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock_utils.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock_utils.py @@ -15,11 +15,21 @@ from __future__ import annotations import json +from os import environ from typing import Callable, Dict, Union from botocore.eventstream import EventStream, EventStreamError from wrapt import ObjectProxy +from opentelemetry._events import Event +from opentelemetry.instrumentation.botocore.environment_variables import ( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, +) +from opentelemetry.semconv._incubating.attributes.gen_ai_attributes import ( + GEN_AI_SYSTEM, + GenAiSystemValues, +) + _StreamDoneCallableT = Callable[[Dict[str, Union[int, str]]], None] _StreamErrorCallableT = Callable[[Exception], None] @@ -220,3 +230,26 @@ def _process_anthropic_claude_chunk(self, chunk): self._process_invocation_metrics(invocation_metrics) self._stream_done_callback(self._response) return + + +def genai_capture_message_content() -> bool: + capture_content = environ.get( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "false" + ) + return capture_content.lower() == "true" + + +def message_to_event(message, capture_content): + attributes = {GEN_AI_SYSTEM: GenAiSystemValues.AWS_BEDROCK.value} + role = message.get("role") + content = message.get("content") + + body = {} + if capture_content and content: + body["content"] = content + + return Event( + name=f"gen_ai.{role}.message", + attributes=attributes, + body=body if body else None, + ) diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/dynamodb.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/dynamodb.py index 1a5f01b6ce..850f32ab69 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/dynamodb.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/dynamodb.py @@ -22,6 +22,7 @@ _AttributeMapT, _AwsSdkCallContext, _AwsSdkExtension, + _BotocoreInstrumentorContext, _BotoResultT, ) from opentelemetry.semconv.trace import DbSystemValues, SpanAttributes @@ -370,7 +371,9 @@ def attr_setter(key: str, value: AttributeValue): def _get_peer_name(self) -> str: return urlparse(self._call_context.endpoint_url).netloc - def before_service_call(self, span: Span): + def before_service_call( + self, span: Span, instrumentor_context: _BotocoreInstrumentorContext + ): if not span.is_recording() or self._op is None: return @@ -380,7 +383,12 @@ def before_service_call(self, span: Span): span.set_attribute, ) - def on_success(self, span: Span, result: _BotoResultT): + def on_success( + self, + span: Span, + result: _BotoResultT, + instrumentor_context: _BotocoreInstrumentorContext, + ): if not span.is_recording(): return diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/lmbd.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/lmbd.py index 57fb8b6794..0d4a1656a3 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/lmbd.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/lmbd.py @@ -22,6 +22,7 @@ _AttributeMapT, _AwsSdkCallContext, _AwsSdkExtension, + _BotocoreInstrumentorContext, ) from opentelemetry.propagate import inject from opentelemetry.semconv.trace import SpanAttributes @@ -119,7 +120,9 @@ def extract_attributes(self, attributes: _AttributeMapT): self._op.extract_attributes(self._call_context, attributes) - def before_service_call(self, span: Span): + def before_service_call( + self, span: Span, instrumentor_context: _BotocoreInstrumentorContext + ): if self._op is None: return diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sns.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sns.py index 9c3df3a2bc..117613603a 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sns.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sns.py @@ -23,6 +23,7 @@ _AttributeMapT, _AwsSdkCallContext, _AwsSdkExtension, + _BotocoreInstrumentorContext, ) from opentelemetry.semconv.trace import ( MessagingDestinationKindValues, @@ -165,6 +166,8 @@ def extract_attributes(self, attributes: _AttributeMapT): if self._op: self._op.extract_attributes(self._call_context, attributes) - def before_service_call(self, span: Span): + def before_service_call( + self, span: Span, instrumentor_context: _BotocoreInstrumentorContext + ): if self._op: self._op.before_service_call(self._call_context, span) diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sqs.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sqs.py index 194e47b57f..79165a0f6c 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sqs.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/sqs.py @@ -16,6 +16,7 @@ from opentelemetry.instrumentation.botocore.extensions.types import ( _AttributeMapT, _AwsSdkExtension, + _BotocoreInstrumentorContext, _BotoResultT, ) from opentelemetry.semconv.trace import SpanAttributes @@ -44,7 +45,12 @@ def extract_attributes(self, attributes: _AttributeMapT): queue_url, ) - def on_success(self, span: Span, result: _BotoResultT): + def on_success( + self, + span: Span, + result: _BotoResultT, + instrumentor_context: _BotocoreInstrumentorContext, + ): operation = self._call_context.operation if operation in _SUPPORTED_OPERATIONS: try: diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/types.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/types.py index 2927c67e93..7de2ac9c23 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/types.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/types.py @@ -15,6 +15,7 @@ import logging from typing import Any, Dict, Optional, Tuple +from opentelemetry._events import EventLogger from opentelemetry.trace import SpanKind from opentelemetry.trace.span import Span from opentelemetry.util.types import AttributeValue @@ -89,10 +90,25 @@ def _get_attr(obj, name: str, default=None): return default +class _BotocoreInstrumentorContext: + def __init__(self, event_logger: EventLogger): + self.event_logger = event_logger + + class _AwsSdkExtension: def __init__(self, call_context: _AwsSdkCallContext): self._call_context = call_context + @staticmethod + def tracer_schema_version() -> str: + """Returns the tracer OTel schema version the extension is following""" + return "1.11.0" + + @staticmethod + def event_logger_schema_version() -> str: + """Returns the event logger OTel schema version the extension is following""" + return "1.30.0" + def should_trace_service_call(self) -> bool: # pylint:disable=no-self-use """Returns if the AWS SDK service call should be traced or not @@ -115,7 +131,9 @@ def extract_attributes(self, attributes: _AttributeMapT): Extensions might override this function to extract additional attributes. """ - def before_service_call(self, span: Span): + def before_service_call( + self, span: Span, instrumentor_context: _BotocoreInstrumentorContext + ): """Callback which gets invoked after the span is created but before the AWS SDK service is called. @@ -123,7 +141,12 @@ def before_service_call(self, span: Span): a carrier. """ - def on_success(self, span: Span, result: _BotoResultT): + def on_success( + self, + span: Span, + result: _BotoResultT, + instrumentor_context: _BotocoreInstrumentorContext, + ): """Callback that gets invoked when the AWS SDK call returns successfully. @@ -131,12 +154,19 @@ def on_success(self, span: Span, result: _BotoResultT): attributes on the span. """ - def on_error(self, span: Span, exception: _BotoClientErrorT): + def on_error( + self, + span: Span, + exception: _BotoClientErrorT, + instrumentor_context: _BotocoreInstrumentorContext, + ): """Callback that gets invoked when the AWS SDK service call raises a ClientError. """ - def after_service_call(self): + def after_service_call( + self, instrumentor_context: _BotocoreInstrumentorContext + ): """Callback that gets invoked after the AWS SDK service was called. Extensions might override this function to do some cleanup tasks. diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/README.md b/instrumentation/opentelemetry-instrumentation-botocore/tests/README.md new file mode 100644 index 0000000000..c2e47a80cd --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/README.md @@ -0,0 +1,11 @@ +## Recording calls + +If you need to record calls you may need to export authentication variables and the default region as environment +variables in order to have the code work properly. +Since tox blocks environment variables by default you need to override its configuration to let them pass: + +``` +export TOX_OVERRIDE=testenv.pass_env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_DEFAULT_REGION +``` + +We are not adding it to tox.ini because of security concerns. diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/bedrock_utils.py b/instrumentation/opentelemetry-instrumentation-botocore/tests/bedrock_utils.py index f3d7f9e5c6..b46f679ddd 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/tests/bedrock_utils.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/bedrock_utils.py @@ -20,6 +20,9 @@ from botocore.response import StreamingBody from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + event_attributes as EventAttributes, +) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) @@ -211,3 +214,43 @@ def assert_all_attributes( GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES, span, ) + + +def remove_none_values(body): + result = {} + for key, value in body.items(): + if value is None: + continue + if isinstance(value, dict): + result[key] = remove_none_values(value) + elif isinstance(value, list): + result[key] = [remove_none_values(i) for i in value] + else: + result[key] = value + return result + + +def assert_log_parent(log, span): + if span: + assert log.log_record.trace_id == span.get_span_context().trace_id + assert log.log_record.span_id == span.get_span_context().span_id + assert ( + log.log_record.trace_flags == span.get_span_context().trace_flags + ) + + +def assert_message_in_logs(log, event_name, expected_content, parent_span): + assert log.log_record.attributes[EventAttributes.EVENT_NAME] == event_name + assert ( + log.log_record.attributes[GenAIAttributes.GEN_AI_SYSTEM] + == GenAIAttributes.GenAiSystemValues.AWS_BEDROCK.value + ) + + if not expected_content: + assert not log.log_record.body + else: + assert log.log_record.body + assert dict(log.log_record.body) == remove_none_values( + expected_content + ) + assert_log_parent(log, parent_span) diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_no_content.yaml b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_no_content.yaml new file mode 100644 index 0000000000..c4d9d7ef4a --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_no_content.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "text": "Say this is a test" + } + ] + } + ], + "inferenceConfig": { + "maxTokens": 10, + "temperature": 0.8, + "topP": 1, + "stopSequences": [ + "|" + ] + } + } + headers: + Content-Length: + - '170' + Content-Type: + - application/json + User-Agent: + - Boto3/1.35.56 md/Botocore#1.35.56 ua/2.0 os/linux#6.1.0-1034-oem md/arch#x86_64 + lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.56 + X-Amz-Date: + - 20250211T141035Z + X-Amz-Security-Token: + - test_aws_security_token + X-Amzn-Trace-Id: + - Root=1-459d61b3-c785d242640df661d315555d;Parent=384fac3cd601ebb0;Sampled=1 + amz-sdk-invocation-id: + - 1de3c974-3f70-4126-85a8-d4b9bcc2e1b1 + amz-sdk-request: + - attempt=1 + authorization: + - Bearer test_aws_authorization + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-lite-v1/converse + response: + body: + string: |- + { + "metrics": { + "latencyMs": 683 + }, + "output": { + "message": { + "content": [ + { + "text": "Hello, how are you doing" + } + ], + "role": "assistant" + } + }, + "stopReason": "max_tokens", + "usage": { + "inputTokens": 8, + "outputTokens": 10, + "totalTokens": 18 + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 11 Feb 2025 14:10:36 GMT + Set-Cookie: test_set_cookie + x-amzn-RequestId: + - ff8f385b-009e-49da-b124-1359bc244f10 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_stream_no_content.yaml b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_stream_no_content.yaml new file mode 100644 index 0000000000..ab3c956855 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_stream_no_content.yaml @@ -0,0 +1,80 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "text": "Say this is a test" + } + ] + } + ], + "inferenceConfig": { + "maxTokens": 10, + "temperature": 0.8, + "topP": 1, + "stopSequences": [ + "|" + ] + } + } + headers: + Content-Length: + - '170' + Content-Type: + - application/json + User-Agent: + - Boto3/1.35.56 md/Botocore#1.35.56 ua/2.0 os/linux#6.1.0-1034-oem md/arch#x86_64 + lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.56 + X-Amz-Date: + - 20250211T141036Z + X-Amz-Security-Token: + - test_aws_security_token + X-Amzn-Trace-Id: + - Root=1-d573e167-b547bec74e267f033a202851;Parent=0a96ae00e1f4d524;Sampled=1 + amz-sdk-invocation-id: + - abc27203-3d67-43ec-b426-9f01928fd698 + amz-sdk-request: + - attempt=1 + authorization: + - Bearer test_aws_authorization + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-lite-v1/converse-stream + response: + body: + string: !!binary | + AAAAuQAAAFL9kIXUCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBh + cHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1u + b3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2NyIsInJvbGUiOiJh + c3Npc3RhbnQifWf51EkAAAC6AAAAV8paC4sLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0 + YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7 + ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiU3VyZSwgZGlkIHlvdSBoYXZl + IGEgcXVlc3Rpb24ifSwicCI6ImFiY2QifQDDASsAAAChAAAAVqptnY4LOmV2ZW50LXR5cGUHABBj + b250ZW50QmxvY2tTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdl + LXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsInAiOiJhYmNkZWZnaGlqa2xtbm9w + cXJzdHV2d3h5ekFCQyJ9AHyeLwAAAIoAAABRghgWOAs6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9w + DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsi + cCI6ImFiY2RlZmciLCJzdG9wUmVhc29uIjoibWF4X3Rva2VucyJ9gceI6AAAAOoAAABOliJsgAs6 + ZXZlbnQtdHlwZQcACG1ldGFkYXRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTpt + ZXNzYWdlLXR5cGUHAAVldmVudHsibWV0cmljcyI6eyJsYXRlbmN5TXMiOjY5Nn0sInAiOiJhYmNk + ZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRIiwidXNhZ2UiOnsiaW5wdXRU + b2tlbnMiOjgsIm91dHB1dFRva2VucyI6MTAsInRvdGFsVG9rZW5zIjoxOH19By4BYA== + headers: + Connection: + - keep-alive + Content-Type: + - application/vnd.amazon.eventstream + Date: + - Tue, 11 Feb 2025 14:10:37 GMT + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + x-amzn-RequestId: + - a2ff4e96-777d-4bf0-9db7-8a9aa7aecf78 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[amazon.nova].yaml b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[amazon.nova].yaml new file mode 100644 index 0000000000..5114d0234a --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[amazon.nova].yaml @@ -0,0 +1,91 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "text": "Say this is a test" + } + ] + } + ], + "inferenceConfig": { + "max_new_tokens": 10, + "temperature": 0.8, + "topP": 1, + "stopSequences": [ + "|" + ] + }, + "schemaVersion": "messages-v1" + } + headers: + Content-Length: + - '207' + User-Agent: + - Boto3/1.35.56 md/Botocore#1.35.56 ua/2.0 os/linux#6.1.0-1034-oem md/arch#x86_64 + lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.56 + X-Amz-Date: + - 20250211T141037Z + X-Amz-Security-Token: + - test_aws_security_token + X-Amzn-Trace-Id: + - Root=1-c2ac5bb7-05fa324e3cb618c2b67e493b;Parent=f005000ccddbf918;Sampled=1 + amz-sdk-invocation-id: + - d988352d-9242-448f-b283-a92382ea97a9 + amz-sdk-request: + - attempt=1 + authorization: + - Bearer test_aws_authorization + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.nova-micro-v1%3A0/invoke + response: + body: + string: |- + { + "output": { + "message": { + "content": [ + { + "text": "It seems like you\u2019re running a test or" + } + ], + "role": "assistant" + } + }, + "stopReason": "max_tokens", + "usage": { + "inputTokens": 5, + "outputTokens": 10, + "totalTokens": 15, + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0 + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 11 Feb 2025 14:10:38 GMT + Set-Cookie: test_set_cookie + X-Amzn-Bedrock-Cache-Read-Input-Token-Count: + - '0' + X-Amzn-Bedrock-Cache-Write-Input-Token-Count: + - '0' + X-Amzn-Bedrock-Input-Token-Count: + - '5' + X-Amzn-Bedrock-Invocation-Latency: + - '223' + X-Amzn-Bedrock-Output-Token-Count: + - '10' + x-amzn-RequestId: + - b302eeb1-efef-4054-9a64-6a343cc2ccf1 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[amazon.titan].yaml b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[amazon.titan].yaml new file mode 100644 index 0000000000..16dc64ee4f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[amazon.titan].yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: |- + { + "inputText": "Say this is a test", + "textGenerationConfig": { + "maxTokenCount": 10, + "temperature": 0.8, + "topP": 1, + "stopSequences": [ + "|" + ] + } + } + headers: + Content-Length: + - '137' + User-Agent: + - Boto3/1.35.56 md/Botocore#1.35.56 ua/2.0 os/linux#6.1.0-1034-oem md/arch#x86_64 + lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.56 + X-Amz-Date: + - 20250211T141038Z + X-Amz-Security-Token: + - test_aws_security_token + X-Amzn-Trace-Id: + - Root=1-9c97c329-c012872e6f21fb9ddf4e7a98;Parent=60362f98aa20fa2d;Sampled=1 + amz-sdk-invocation-id: + - 8feb1f58-f561-497c-9b7e-8050c3d1fd88 + amz-sdk-request: + - attempt=1 + authorization: + - Bearer test_aws_authorization + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-lite-v1/invoke + response: + body: + string: |- + { + "inputTextTokenCount": 5, + "results": [ + { + "tokenCount": 10, + "outputText": "\nHello! I am a computer program designed to", + "completionReason": "LENGTH" + } + ] + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 11 Feb 2025 14:10:39 GMT + Set-Cookie: test_set_cookie + X-Amzn-Bedrock-Input-Token-Count: + - '5' + X-Amzn-Bedrock-Invocation-Latency: + - '623' + X-Amzn-Bedrock-Output-Token-Count: + - '10' + x-amzn-RequestId: + - 25260e3f-4a5f-47aa-afe2-99bca8f9bf68 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[anthropic.claude].yaml b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[anthropic.claude].yaml new file mode 100644 index 0000000000..29bd9fd5a0 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content[anthropic.claude].yaml @@ -0,0 +1,84 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "text": "Say this is a test", + "type": "text" + } + ] + } + ], + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 10, + "temperature": 0.8, + "top_p": 1, + "stop_sequences": [ + "|" + ] + } + headers: + Content-Length: + - '211' + User-Agent: + - Boto3/1.35.56 md/Botocore#1.35.56 ua/2.0 os/linux#6.1.0-1034-oem md/arch#x86_64 + lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.56 + X-Amz-Date: + - 20250211T141039Z + X-Amz-Security-Token: + - test_aws_security_token + X-Amzn-Trace-Id: + - Root=1-8be6f3ea-614b53957b659455b7f8a1ff;Parent=9f44c46d2e6821c0;Sampled=1 + amz-sdk-invocation-id: + - 5d18ca20-a5ac-4403-99b5-aaa6ecf46b5d + amz-sdk-request: + - attempt=1 + authorization: + - Bearer test_aws_authorization + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/invoke + response: + body: + string: |- + { + "id": "msg_bdrk_01QBSj9wkkgvNKMZZxMa6J77", + "type": "message", + "role": "assistant", + "model": "claude-2.0", + "content": [ + { + "type": "text", + "text": "Okay, I just said \"This is a test" + } + ], + "stop_reason": "max_tokens", + "stop_sequence": null, + "usage": { + "input_tokens": 14, + "output_tokens": 10 + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 11 Feb 2025 14:10:40 GMT + Set-Cookie: test_set_cookie + X-Amzn-Bedrock-Input-Token-Count: + - '14' + X-Amzn-Bedrock-Invocation-Latency: + - '506' + X-Amzn-Bedrock-Output-Token-Count: + - '10' + x-amzn-RequestId: + - a6cbb4fd-4a3f-4c19-8e68-23ad1622ddfe + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_bedrock.py b/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_bedrock.py index f277ba895e..24a30eedc7 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_bedrock.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_bedrock.py @@ -29,12 +29,17 @@ from .bedrock_utils import ( assert_completion_attributes_from_streaming_body, assert_converse_completion_attributes, + assert_message_in_logs, assert_stream_completion_attributes, ) BOTO3_VERSION = tuple(int(x) for x in boto3.__version__.split(".")) +def filter_message_keys(message, keys): + return {k: v for k, v in message.items() if k in keys} + + @pytest.mark.skipif( BOTO3_VERSION < (1, 35, 56), reason="Converse API not available" ) @@ -73,7 +78,51 @@ def test_converse_with_content( ) logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + assert len(logs) == 1 + user_content = filter_message_keys(messages[0], ["content"]) + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) + + +@pytest.mark.skipif( + BOTO3_VERSION < (1, 35, 56), reason="Converse API not available" +) +@pytest.mark.vcr() +def test_converse_no_content( + span_exporter, + log_exporter, + bedrock_runtime_client, + instrument_no_content, +): + messages = [{"role": "user", "content": [{"text": "Say this is a test"}]}] + + llm_model_value = "amazon.titan-text-lite-v1" + max_tokens, temperature, top_p, stop_sequences = 10, 0.8, 1, ["|"] + response = bedrock_runtime_client.converse( + messages=messages, + modelId=llm_model_value, + inferenceConfig={ + "maxTokens": max_tokens, + "temperature": temperature, + "topP": top_p, + "stopSequences": stop_sequences, + }, + ) + + (span,) = span_exporter.get_finished_spans() + assert_converse_completion_attributes( + span, + llm_model_value, + response, + "chat", + top_p, + temperature, + max_tokens, + stop_sequences, + ) + + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + assert_message_in_logs(logs[0], "gen_ai.user.message", None, span) @pytest.mark.skipif( @@ -107,7 +156,8 @@ def test_converse_with_invalid_model( assert span.attributes[ERROR_TYPE] == "ValidationException" logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + user_content = filter_message_keys(messages[0], ["content"]) + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) @pytest.mark.skipif( @@ -170,7 +220,73 @@ def test_converse_stream_with_content( ) logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + assert len(logs) == 1 + user_content = filter_message_keys(messages[0], ["content"]) + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) + + +@pytest.mark.skipif( + BOTO3_VERSION < (1, 35, 56), reason="ConverseStream API not available" +) +@pytest.mark.vcr() +def test_converse_stream_no_content( + span_exporter, + log_exporter, + bedrock_runtime_client, + instrument_no_content, +): + # pylint:disable=too-many-locals + messages = [{"role": "user", "content": [{"text": "Say this is a test"}]}] + + llm_model_value = "amazon.titan-text-lite-v1" + max_tokens, temperature, top_p, stop_sequences = 10, 0.8, 1, ["|"] + response = bedrock_runtime_client.converse_stream( + messages=messages, + modelId=llm_model_value, + inferenceConfig={ + "maxTokens": max_tokens, + "temperature": temperature, + "topP": top_p, + "stopSequences": stop_sequences, + }, + ) + + # consume the stream in order to have it traced + finish_reason = None + input_tokens, output_tokens = None, None + text = "" + for event in response["stream"]: + if "contentBlockDelta" in event: + text += event["contentBlockDelta"]["delta"]["text"] + if "messageStop" in event: + finish_reason = (event["messageStop"]["stopReason"],) + if "metadata" in event: + usage = event["metadata"]["usage"] + input_tokens = usage["inputTokens"] + output_tokens = usage["outputTokens"] + + assert text + assert finish_reason + assert input_tokens + assert output_tokens + + (span,) = span_exporter.get_finished_spans() + assert_stream_completion_attributes( + span, + llm_model_value, + input_tokens, + output_tokens, + finish_reason, + "chat", + top_p, + temperature, + max_tokens, + stop_sequences, + ) + + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + assert_message_in_logs(logs[0], "gen_ai.user.message", None, span) @pytest.mark.skipif( @@ -229,7 +345,9 @@ def test_converse_stream_handles_event_stream_error( assert span.attributes[ERROR_TYPE] == "EventStreamError" logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + assert len(logs) == 1 + user_content = filter_message_keys(messages[0], ["content"]) + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) @pytest.mark.skipif( @@ -262,7 +380,9 @@ def test_converse_stream_with_invalid_model( assert span.attributes[ERROR_TYPE] == "ValidationException" logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + assert len(logs) == 1 + user_content = filter_message_keys(messages[0], ["content"]) + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) def get_invoke_model_body( @@ -356,7 +476,53 @@ def test_invoke_model_with_content( ) logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + assert len(logs) == 1 + if model_family == "anthropic.claude": + user_content = { + "content": [{"text": "Say this is a test", "type": "text"}] + } + else: + user_content = {"content": [{"text": "Say this is a test"}]} + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) + + +@pytest.mark.parametrize( + "model_family", + ["amazon.nova", "amazon.titan", "anthropic.claude"], +) +@pytest.mark.vcr() +def test_invoke_model_no_content( + span_exporter, + log_exporter, + bedrock_runtime_client, + instrument_no_content, + model_family, +): + llm_model_value = get_model_name_from_family(model_family) + max_tokens, temperature, top_p, stop_sequences = 10, 0.8, 1, ["|"] + body = get_invoke_model_body( + llm_model_value, max_tokens, temperature, top_p, stop_sequences + ) + response = bedrock_runtime_client.invoke_model( + body=body, + modelId=llm_model_value, + ) + + (span,) = span_exporter.get_finished_spans() + assert_completion_attributes_from_streaming_body( + span, + llm_model_value, + response, + "text_completion" if model_family == "amazon.titan" else "chat", + top_p, + temperature, + max_tokens, + stop_sequences, + ) + + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + assert_message_in_logs(logs[0], "gen_ai.user.message", None, span) @pytest.mark.vcr() @@ -400,7 +566,7 @@ def test_invoke_model_with_response_stream_with_content( instrument_with_content, model_family, ): - # pylint:disable=too-many-locals + # pylint:disable=too-many-locals,too-many-branches llm_model_value = get_model_name_from_family(model_family) max_tokens, temperature, top_p, stop_sequences = 10, 0.8, 1, ["|"] body = get_invoke_model_body( @@ -471,7 +637,14 @@ def test_invoke_model_with_response_stream_with_content( ) logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + assert len(logs) == 1 + if model_family == "anthropic.claude": + user_content = { + "content": [{"text": "Say this is a test", "type": "text"}] + } + else: + user_content = {"content": [{"text": "Say this is a test"}]} + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) @pytest.mark.vcr() @@ -521,7 +694,9 @@ def test_invoke_model_with_response_stream_handles_stream_error( ) logs = log_exporter.get_finished_logs() - assert len(logs) == 0 + assert len(logs) == 1 + user_content = {"content": [{"text": "Say this is a test"}]} + assert_message_in_logs(logs[0], "gen_ai.user.message", user_content, span) @pytest.mark.vcr() diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_dynamodb.py b/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_dynamodb.py index 2240baff3a..de4fc72153 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_dynamodb.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_dynamodb.py @@ -20,6 +20,7 @@ from opentelemetry.instrumentation.botocore import BotocoreInstrumentor from opentelemetry.instrumentation.botocore.extensions.dynamodb import ( + _BotocoreInstrumentorContext, _DynamoDbExtension, ) from opentelemetry.semconv.trace import SpanAttributes @@ -180,7 +181,9 @@ def assert_extension_item_col_metrics(self, operation: str): extension = self._create_extension(operation) extension.on_success( - span, {"ItemCollectionMetrics": {"ItemCollectionKey": {"id": "1"}}} + span, + {"ItemCollectionMetrics": {"ItemCollectionKey": {"id": "1"}}}, + _BotocoreInstrumentorContext(event_logger=mock.Mock()), ) self.assert_item_col_metrics(span) @@ -290,7 +293,9 @@ def test_delete_item_consumed_capacity(self): extension = self._create_extension("DeleteItem") extension.on_success( - span, {"ConsumedCapacity": {"TableName": "table"}} + span, + {"ConsumedCapacity": {"TableName": "table"}}, + _BotocoreInstrumentorContext(event_logger=mock.Mock()), ) self.assert_consumed_capacity(span, "table") diff --git a/tox.ini b/tox.ini index b45b6b4d4c..c12cf0a681 100644 --- a/tox.ini +++ b/tox.ini @@ -421,11 +421,6 @@ test_deps = opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils -pass_env = - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN - AWS_DEFAULT_REGION deps = lint: -r dev-requirements.txt