Merge branch 'main' into dependabot/pip/requests-2.32.4

Author: majanjua-amzn

.github/workflows/release-lambda.yml

@@ -9,10 +9,10 @@ on:
       aws_region:
         description: 'Deploy to aws regions'
         required: true
-        default: 'us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, af-south-1, ap-east-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, il-central-1, me-central-1, me-south-1'
+        default: 'us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, af-south-1, ap-east-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, il-central-1, me-central-1, me-south-1, ap-southeast-5, ap-southeast-7, mx-central-1, ca-west-1, cn-north-1, cn-northwest-1'
 
 env:
-  COMMERCIAL_REGIONS: us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1
+  COMMERCIAL_REGIONS: us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, ap-southeast-5, ap-southeast-7, mx-central-1, ca-west-1, cn-north-1, cn-northwest-1
   LAYER_NAME: AWSOpenTelemetryDistroPython
 
 permissions:

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py

@@ -33,3 +33,4 @@
 AWS_LAMBDA_FUNCTION_NAME: str = "aws.lambda.function.name"
 AWS_LAMBDA_RESOURCEMAPPING_ID: str = "aws.lambda.resource_mapping.id"
 AWS_LAMBDA_FUNCTION_ARN: str = "aws.lambda.function.arn"
+AWS_SERVICE_TYPE: str = "aws.service.type"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -35,3 +35,32 @@ def is_installed(req: str) -> bool:
 def is_agent_observability_enabled() -> bool:
     """Is the Agentic AI monitoring flag set to true?"""
     return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"
+
+
+def get_aws_region() -> str:
+    """Get AWS region using botocore session.
+
+    botocore automatically checks in the following priority order:
+    1. AWS_REGION environment variable
+    2. AWS_DEFAULT_REGION environment variable
+    3. AWS CLI config file (~/.aws/config)
+    4. EC2 instance metadata service
+
+    Returns:
+        The AWS region if found, None otherwise.
+    """
+    if is_installed("botocore"):
+        try:
+            from botocore import session  # pylint: disable=import-outside-toplevel
+
+            botocore_session = session.Session()
+            if botocore_session.region_name:
+                return botocore_session.region_name
+        except (ImportError, AttributeError):
+            # botocore failed to determine region
+            pass
+
+    _logger.warning(
+        "AWS region not found. Please set AWS_REGION environment variable or configure AWS CLI with 'aws configure'."
+    )
+    return None

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -4,14 +4,14 @@
 import os
 import re
 from logging import NOTSET, Logger, getLogger
-from typing import ClassVar, Dict, List, Type, Union
+from typing import ClassVar, Dict, List, NamedTuple, Optional, Type, Union
 
 from importlib_metadata import version
 from typing_extensions import override
 
-from amazon.opentelemetry.distro._aws_attribute_keys import AWS_LOCAL_SERVICE
+from amazon.opentelemetry.distro._aws_attribute_keys import AWS_LOCAL_SERVICE, AWS_SERVICE_TYPE
 from amazon.opentelemetry.distro._aws_resource_attribute_configurator import get_service_attribute
-from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled, is_installed
 from amazon.opentelemetry.distro.always_record_sampler import AlwaysRecordSampler
 from amazon.opentelemetry.distro.attribute_propagating_span_processor_builder import (
     AttributePropagatingSpanProcessorBuilder,
@@ -98,13 +98,29 @@
 
 AWS_OTLP_LOGS_GROUP_HEADER = "x-aws-log-group"
 AWS_OTLP_LOGS_STREAM_HEADER = "x-aws-log-stream"
+AWS_EMF_METRICS_NAMESPACE = "x-aws-metric-namespace"
 
 # UDP package size is not larger than 64KB
 LAMBDA_SPAN_EXPORT_BATCH_SIZE = 10
 
+OTEL_TRACES_EXPORTER = "OTEL_TRACES_EXPORTER"
+OTEL_LOGS_EXPORTER = "OTEL_LOGS_EXPORTER"
+OTEL_METRICS_EXPORTER = "OTEL_METRICS_EXPORTER"
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+OTEL_TRACES_SAMPLER = "OTEL_TRACES_SAMPLER"
+OTEL_PYTHON_DISABLED_INSTRUMENTATIONS = "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS"
+OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED = "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"
+
 _logger: Logger = getLogger(__name__)
 
 
+class OtlpLogHeaderSetting(NamedTuple):
+    log_group: Optional[str]
+    log_stream: Optional[str]
+    namespace: Optional[str]
+    is_valid: bool
+
+
 class AwsOpenTelemetryConfigurator(_OTelSDKConfigurator):
     """
     This AwsOpenTelemetryConfigurator extend _OTelSDKConfigurator configuration with the following change:
@@ -133,6 +149,11 @@ def _configure(self, **kwargs):
 # Long term, we wish to contribute this to upstream to improve initialization customizability and reduce dependency on
 # internal logic.
 def _initialize_components():
+    # Remove 'awsemf' from OTEL_METRICS_EXPORTER if present to prevent validation errors
+    # from _import_exporters in OTel dependencies which would try to load exporters
+    # We will contribute emf exporter to upstream for supporting OTel metrics in SDK
+    is_emf_enabled = _check_emf_exporter_enabled()
+
     trace_exporters, metric_exporters, log_exporters = _import_exporters(
         _get_exporter_names("traces"),
         _get_exporter_names("metrics"),
@@ -153,7 +174,7 @@ def _initialize_components():
             AwsEksResourceDetector(),
             AwsEcsResourceDetector(),
         ]
-        if not _is_lambda_environment()
+        if not (_is_lambda_environment() or is_agent_observability_enabled())
         else []
     )
 
@@ -168,7 +189,8 @@ def _initialize_components():
         sampler=sampler,
         resource=resource,
     )
-    _init_metrics(metric_exporters, resource)
+
+    _init_metrics(metric_exporters, resource, is_emf_enabled)
     logging_enabled = os.getenv(_OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "false")
     if logging_enabled.strip().lower() == "true":
         _init_logging(log_exporters, resource)
@@ -227,6 +249,7 @@ def _init_tracing(
 def _init_metrics(
     exporters_or_readers: Dict[str, Union[Type[MetricExporter], Type[MetricReader]]],
     resource: Resource = None,
+    is_emf_enabled: bool = False,
 ):
     metric_readers = []
     views = []
@@ -239,7 +262,7 @@ def _init_metrics(
         else:
             metric_readers.append(PeriodicExportingMetricReader(exporter_or_reader_class(**exporter_args)))
 
-    _customize_metric_exporters(metric_readers, views)
+    _customize_metric_exporters(metric_readers, views, is_emf_enabled)
 
     provider = MeterProvider(resource=resource, metric_readers=metric_readers, views=views)
     set_meter_provider(provider)
@@ -265,6 +288,17 @@ def _export_unsampled_span_for_lambda(trace_provider: TracerProvider, resource:
     )
 
 
+def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvider, resource: Resource = None):
+    if not is_agent_observability_enabled():
+        return
+
+    traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT)
+
+    span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint, logger_provider=get_logger_provider())
+
+    trace_provider.add_span_processor(BatchUnsampledSpanProcessor(span_exporter=span_exporter))
+
+
 def _is_defer_to_workers_enabled():
     return os.environ.get(OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG, "false").strip().lower() == "true"
 
@@ -389,7 +423,7 @@ def _customize_logs_exporter(log_exporter: LogExporter, resource: Resource) -> L
     if _is_aws_otlp_endpoint(logs_endpoint, "logs"):
         _logger.info("Detected using AWS OTLP Logs Endpoint.")
 
-        if isinstance(log_exporter, OTLPLogExporter) and _validate_logs_headers():
+        if isinstance(log_exporter, OTLPLogExporter) and _validate_and_fetch_logs_header().is_valid:
             # Setting default compression mode to Gzip as this is the behavior in upstream's
             # collector otlp http exporter:
             # https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter
@@ -408,9 +442,14 @@ def _customize_span_processors(provider: TracerProvider, resource: Resource) ->
     if _is_lambda_environment():
         provider.add_span_processor(AwsLambdaSpanProcessor())
 
+    # We always send 100% spans to Genesis platform for agent observability because
+    # AI applications typically have low throughput traffic patterns and require
+    # comprehensive monitoring to catch subtle failure modes like hallucinations
+    # and quality degradation that sampling could miss.
     # Add session.id baggage attribute to span attributes to support AI Agent use cases
     # enabling session ID tracking in spans.
     if is_agent_observability_enabled():
+        _export_unsampled_span_for_agent_observability(provider, resource)
 
         def session_id_predicate(baggage_key: str) -> bool:
             return baggage_key == "session.id"
@@ -442,7 +481,9 @@ def session_id_predicate(baggage_key: str) -> bool:
     return
 
 
-def _customize_metric_exporters(metric_readers: List[MetricReader], views: List[View]) -> None:
+def _customize_metric_exporters(
+    metric_readers: List[MetricReader], views: List[View], is_emf_enabled: bool = False
+) -> None:
     if _is_application_signals_runtime_enabled():
         _get_runtime_metric_views(views, 0 == len(metric_readers))
 
@@ -454,6 +495,11 @@ def _customize_metric_exporters(metric_readers: List[MetricReader], views: List[
         )
         metric_readers.append(scope_based_periodic_exporting_metric_reader)
 
+    if is_emf_enabled:
+        emf_exporter = create_emf_exporter()
+        if emf_exporter:
+            metric_readers.append(PeriodicExportingMetricReader(emf_exporter))
+
 
 def _get_runtime_metric_views(views: List[View], retain_runtime_only: bool) -> None:
     runtime_metrics_scope_name = SYSTEM_METRICS_INSTRUMENTATION_SCOPE_NAME
@@ -509,7 +555,15 @@ def _customize_resource(resource: Resource) -> Resource:
     if is_unknown:
         _logger.debug("No valid service name found")
 
-    return resource.merge(Resource.create({AWS_LOCAL_SERVICE: service_name}))
+    custom_attributes = {AWS_LOCAL_SERVICE: service_name}
+
+    if is_agent_observability_enabled():
+        # Add aws.service.type if it doesn't exist in the resource
+        if resource and resource.attributes.get(AWS_SERVICE_TYPE) is None:
+            # Set a default agent type for AI agent observability
+            custom_attributes[AWS_SERVICE_TYPE] = "gen_ai_agent"
+
+    return resource.merge(Resource.create(custom_attributes))
 
 
 def _is_application_signals_enabled():
@@ -543,7 +597,7 @@ def _is_aws_otlp_endpoint(otlp_endpoint: str = None, service: str = "xray") -> b
     return bool(re.match(pattern, otlp_endpoint.lower()))
 
 
-def _validate_logs_headers() -> bool:
+def _validate_and_fetch_logs_header() -> OtlpLogHeaderSetting:
     """Checks if x-aws-log-group and x-aws-log-stream are present in the headers in order to send logs to
     AWS OTLP Logs endpoint."""
 
@@ -554,26 +608,36 @@ def _validate_logs_headers() -> bool:
             "Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS "
             "to include x-aws-log-group and x-aws-log-stream"
         )
-        return False
+        return OtlpLogHeaderSetting(None, None, None, False)
 
+    log_group = None
+    log_stream = None
+    namespace = None
     filtered_log_headers_count = 0
 
     for pair in logs_headers.split(","):
         if "=" in pair:
             split = pair.split("=", 1)
             key = split[0]
             value = split[1]
-            if key in (AWS_OTLP_LOGS_GROUP_HEADER, AWS_OTLP_LOGS_STREAM_HEADER) and value:
+            if key == AWS_OTLP_LOGS_GROUP_HEADER and value:
+                log_group = value
+                filtered_log_headers_count += 1
+            elif key == AWS_OTLP_LOGS_STREAM_HEADER and value:
+                log_stream = value
                 filtered_log_headers_count += 1
+            elif key == AWS_EMF_METRICS_NAMESPACE and value:
+                namespace = value
 
-    if filtered_log_headers_count != 2:
+    is_valid = filtered_log_headers_count == 2 and log_group is not None and log_stream is not None
+
+    if not is_valid:
         _logger.warning(
             "Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS "
             "to have values for x-aws-log-group and x-aws-log-stream"
         )
-        return False
 
-    return True
+    return OtlpLogHeaderSetting(log_group, log_stream, namespace, is_valid)
 
 
 def _get_metric_export_interval():
@@ -644,3 +708,73 @@ def create_exporter(self):
             )
 
         raise RuntimeError(f"Unsupported AWS Application Signals export protocol: {protocol} ")
+
+
+def _check_emf_exporter_enabled() -> bool:
+    """
+    Checks if OTEL_METRICS_EXPORTER contains "awsemf", removes it if present,
+    and updates the environment variable.
+
+    Remove 'awsemf' from OTEL_METRICS_EXPORTER if present to prevent validation errors
+    from _import_exporters in OTel dependencies which would try to load exporters
+    We will contribute emf exporter to upstream for supporting OTel metrics in SDK
+
+    Returns:
+    bool: True if "awsemf" was found and removed, False otherwise.
+    """
+    # Get the current exporter value
+    exporter_value = os.environ.get("OTEL_METRICS_EXPORTER", "")
+
+    # Check if it's empty
+    if not exporter_value:
+        return False
+
+    # Split by comma and convert to list
+    exporters = [exp.strip() for exp in exporter_value.split(",")]
+
+    # Check if awsemf is in the list
+    if "awsemf" not in exporters:
+        return False
+
+    # Remove awsemf from the list
+    exporters.remove("awsemf")
+
+    # Join the remaining exporters and update the environment variable
+    new_value = ",".join(exporters) if exporters else ""
+
+    # Set the new value (or unset if empty)
+    if new_value:
+        os.environ["OTEL_METRICS_EXPORTER"] = new_value
+    elif "OTEL_METRICS_EXPORTER" in os.environ:
+        del os.environ["OTEL_METRICS_EXPORTER"]
+
+    return True
+
+
+def create_emf_exporter():
+    """Create and configure the CloudWatch EMF exporter."""
+    try:
+        # Check if botocore is available before importing the EMF exporter
+        if not is_installed("botocore"):
+            _logger.warning("botocore is not installed. EMF exporter requires botocore")
+            return None
+
+        # pylint: disable=import-outside-toplevel
+        from amazon.opentelemetry.distro.exporter.aws.metrics.aws_cloudwatch_emf_exporter import (
+            AwsCloudWatchEmfExporter,
+        )
+
+        log_header_setting = _validate_and_fetch_logs_header()
+
+        if not log_header_setting.is_valid:
+            return None
+
+        return AwsCloudWatchEmfExporter(
+            namespace=log_header_setting.namespace,
+            log_group_name=log_header_setting.log_group,
+            log_stream_name=log_header_setting.log_stream,
+        )
+    # pylint: disable=broad-exception-caught
+    except Exception as errors:
+        _logger.error("Failed to create EMF exporter: %s", errors)
+        return None