Merge remote-tracking branch 'upstream/main' into logs-mainline

Author: liustve

.github/workflows/release-lambda.yml

@@ -9,7 +9,7 @@ on:
       aws_region:
         description: 'Deploy to aws regions'
         required: true
-        default: 'us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, af-south-1, ap-east-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, il-central-1, me-central-1, me-south-1'
+        default: 'us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, af-south-1, ap-east-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, il-central-1, me-central-1, me-south-1, ap-southeast-5, ap-southeast-7, mx-central-1, ca-west-1, cn-north-1, cn-northwest-1'
 
 env:
   COMMERCIAL_REGIONS: us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, ap-southeast-5, ap-southeast-7, mx-central-1, ca-west-1, cn-north-1, cn-northwest-1

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -35,3 +35,32 @@ def is_installed(req: str) -> bool:
 def is_agent_observability_enabled() -> bool:
     """Is the Agentic AI monitoring flag set to true?"""
     return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"
+
+
+def get_aws_region() -> str:
+    """Get AWS region using botocore session.
+
+    botocore automatically checks in the following priority order:
+    1. AWS_REGION environment variable
+    2. AWS_DEFAULT_REGION environment variable
+    3. AWS CLI config file (~/.aws/config)
+    4. EC2 instance metadata service
+
+    Returns:
+        The AWS region if found, None otherwise.
+    """
+    if is_installed("botocore"):
+        try:
+            from botocore import session  # pylint: disable=import-outside-toplevel
+
+            botocore_session = session.Session()
+            if botocore_session.region_name:
+                return botocore_session.region_name
+        except (ImportError, AttributeError):
+            # botocore failed to determine region
+            pass
+
+    _logger.warning(
+        "AWS region not found. Please set AWS_REGION environment variable or configure AWS CLI with 'aws configure'."
+    )
+    return None

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -4,14 +4,15 @@
 import logging
 import os
 import re
-from typing import ClassVar, Dict, List, Optional, Type, Union
+from logging import NOTSET, Logger, getLogger
+from typing import ClassVar, Dict, List, NamedTuple, Optional, Type, Union
 
 from importlib_metadata import version
 from typing_extensions import override
 
 from amazon.opentelemetry.distro._aws_attribute_keys import AWS_LOCAL_SERVICE
 from amazon.opentelemetry.distro._aws_resource_attribute_configurator import get_service_attribute
-from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled, is_installed
 from amazon.opentelemetry.distro.always_record_sampler import AlwaysRecordSampler
 from amazon.opentelemetry.distro.attribute_propagating_span_processor_builder import (
     AttributePropagatingSpanProcessorBuilder,
@@ -104,11 +105,27 @@
 
 AWS_OTLP_LOGS_GROUP_HEADER = "x-aws-log-group"
 AWS_OTLP_LOGS_STREAM_HEADER = "x-aws-log-stream"
+AWS_EMF_METRICS_NAMESPACE = "x-aws-metric-namespace"
 
 # UDP package size is not larger than 64KB
 LAMBDA_SPAN_EXPORT_BATCH_SIZE = 10
 
-_logger: logging.Logger = logging.getLogger(__name__)
+OTEL_TRACES_EXPORTER = "OTEL_TRACES_EXPORTER"
+OTEL_LOGS_EXPORTER = "OTEL_LOGS_EXPORTER"
+OTEL_METRICS_EXPORTER = "OTEL_METRICS_EXPORTER"
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+OTEL_TRACES_SAMPLER = "OTEL_TRACES_SAMPLER"
+OTEL_PYTHON_DISABLED_INSTRUMENTATIONS = "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS"
+OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED = "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"
+
+_logger: Logger = getLogger(__name__)
+
+
+class OtlpLogHeaderSetting(NamedTuple):
+    log_group: Optional[str]
+    log_stream: Optional[str]
+    namespace: Optional[str]
+    is_valid: bool
 
 
 class AwsOpenTelemetryConfigurator(_OTelSDKConfigurator):
@@ -138,7 +155,12 @@ def _configure(self, **kwargs):
 # The OpenTelemetry Authors code
 # Long term, we wish to contribute this to upstream to improve initialization customizability and reduce dependency on
 # internal logic.
-def _initialize_components(setup_logging_handler: Optional[bool] = None):
+def _initialize_components():
+    # Remove 'awsemf' from OTEL_METRICS_EXPORTER if present to prevent validation errors
+    # from _import_exporters in OTel dependencies which would try to load exporters
+    # We will contribute emf exporter to upstream for supporting OTel metrics in SDK
+    is_emf_enabled = _check_emf_exporter_enabled()
+
     trace_exporters, metric_exporters, log_exporters = _import_exporters(
         _get_exporter_names("traces"),
         _get_exporter_names("metrics"),
@@ -174,13 +196,11 @@ def _initialize_components(setup_logging_handler: Optional[bool] = None):
         sampler=sampler,
         resource=resource,
     )
-    _init_metrics(metric_exporters, resource)
 
-    if setup_logging_handler is None:
-        setup_logging_handler = (
-            os.getenv(_OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "false").strip().lower() == "true"
-        )
-        _init_logging(log_exporters, resource, setup_logging_handler)
+    _init_metrics(metric_exporters, resource, is_emf_enabled)
+    logging_enabled = os.getenv(_OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "false")
+    if logging_enabled.strip().lower() == "true":
+        _init_logging(log_exporters, resource)
 
 
 def _init_logging(
@@ -238,6 +258,7 @@ def _init_tracing(
 def _init_metrics(
     exporters_or_readers: Dict[str, Union[Type[MetricExporter], Type[MetricReader]]],
     resource: Resource = None,
+    is_emf_enabled: bool = False,
 ):
     metric_readers = []
     views = []
@@ -250,7 +271,7 @@ def _init_metrics(
         else:
             metric_readers.append(PeriodicExportingMetricReader(exporter_or_reader_class(**exporter_args)))
 
-    _customize_metric_exporters(metric_readers, views)
+    _customize_metric_exporters(metric_readers, views, is_emf_enabled)
 
     provider = MeterProvider(resource=resource, metric_readers=metric_readers, views=views)
     set_meter_provider(provider)
@@ -276,6 +297,17 @@ def _export_unsampled_span_for_lambda(trace_provider: TracerProvider, resource:
     )
 
 
+def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvider, resource: Resource = None):
+    if not is_agent_observability_enabled():
+        return
+
+    traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT)
+
+    span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint, logger_provider=get_logger_provider())
+
+    trace_provider.add_span_processor(BatchUnsampledSpanProcessor(span_exporter=span_exporter))
+
+
 def _is_defer_to_workers_enabled():
     return os.environ.get(OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG, "false").strip().lower() == "true"
 
@@ -407,7 +439,7 @@ def _customize_logs_exporter(log_exporter: LogExporter) -> LogExporter:
     if _is_aws_otlp_endpoint(logs_endpoint, "logs"):
         _logger.info("Detected using AWS OTLP Logs Endpoint.")
 
-        if isinstance(log_exporter, OTLPLogExporter) and _validate_logs_headers():
+        if isinstance(log_exporter, OTLPLogExporter) and _validate_and_fetch_logs_header().is_valid:
             # Setting default compression mode to Gzip as this is the behavior in upstream's
             # collector otlp http exporter:
             # https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter
@@ -426,9 +458,14 @@ def _customize_span_processors(provider: TracerProvider, resource: Resource) ->
     if _is_lambda_environment():
         provider.add_span_processor(AwsLambdaSpanProcessor())
 
+    # We always send 100% spans to Genesis platform for agent observability because
+    # AI applications typically have low throughput traffic patterns and require
+    # comprehensive monitoring to catch subtle failure modes like hallucinations
+    # and quality degradation that sampling could miss.
     # Add session.id baggage attribute to span attributes to support AI Agent use cases
     # enabling session ID tracking in spans.
     if is_agent_observability_enabled():
+        _export_unsampled_span_for_agent_observability(provider, resource)
 
         def session_id_predicate(baggage_key: str) -> bool:
             return baggage_key == "session.id"
@@ -460,7 +497,9 @@ def session_id_predicate(baggage_key: str) -> bool:
     return
 
 
-def _customize_metric_exporters(metric_readers: List[MetricReader], views: List[View]) -> None:
+def _customize_metric_exporters(
+    metric_readers: List[MetricReader], views: List[View], is_emf_enabled: bool = False
+) -> None:
     if _is_application_signals_runtime_enabled():
         _get_runtime_metric_views(views, 0 == len(metric_readers))
 
@@ -472,6 +511,11 @@ def _customize_metric_exporters(metric_readers: List[MetricReader], views: List[
         )
         metric_readers.append(scope_based_periodic_exporting_metric_reader)
 
+    if is_emf_enabled:
+        emf_exporter = create_emf_exporter()
+        if emf_exporter:
+            metric_readers.append(PeriodicExportingMetricReader(emf_exporter))
+
 
 def _get_runtime_metric_views(views: List[View], retain_runtime_only: bool) -> None:
     runtime_metrics_scope_name = SYSTEM_METRICS_INSTRUMENTATION_SCOPE_NAME
@@ -561,7 +605,7 @@ def _is_aws_otlp_endpoint(otlp_endpoint: Optional[str] = None, service: str = "x
     return bool(re.match(pattern, otlp_endpoint.lower()))
 
 
-def _validate_logs_headers() -> bool:
+def _validate_and_fetch_logs_header() -> OtlpLogHeaderSetting:
     """Checks if x-aws-log-group and x-aws-log-stream are present in the headers in order to send logs to
     AWS OTLP Logs endpoint."""
 
@@ -572,26 +616,36 @@ def _validate_logs_headers() -> bool:
             "Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS "
             "to include x-aws-log-group and x-aws-log-stream"
         )
-        return False
+        return OtlpLogHeaderSetting(None, None, None, False)
 
+    log_group = None
+    log_stream = None
+    namespace = None
     filtered_log_headers_count = 0
 
     for pair in logs_headers.split(","):
         if "=" in pair:
             split = pair.split("=", 1)
             key = split[0]
             value = split[1]
-            if key in (AWS_OTLP_LOGS_GROUP_HEADER, AWS_OTLP_LOGS_STREAM_HEADER) and value:
+            if key == AWS_OTLP_LOGS_GROUP_HEADER and value:
+                log_group = value
                 filtered_log_headers_count += 1
+            elif key == AWS_OTLP_LOGS_STREAM_HEADER and value:
+                log_stream = value
+                filtered_log_headers_count += 1
+            elif key == AWS_EMF_METRICS_NAMESPACE and value:
+                namespace = value
+
+    is_valid = filtered_log_headers_count == 2 and log_group is not None and log_stream is not None
 
-    if filtered_log_headers_count != 2:
+    if not is_valid:
         _logger.warning(
             "Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS "
             "to have values for x-aws-log-group and x-aws-log-stream"
         )
-        return False
 
-    return True
+    return OtlpLogHeaderSetting(log_group, log_stream, namespace, is_valid)
 
 
 def _get_metric_export_interval():
@@ -662,3 +716,73 @@ def create_exporter(self):
             )
 
         raise RuntimeError(f"Unsupported AWS Application Signals export protocol: {protocol} ")
+
+
+def _check_emf_exporter_enabled() -> bool:
+    """
+    Checks if OTEL_METRICS_EXPORTER contains "awsemf", removes it if present,
+    and updates the environment variable.
+
+    Remove 'awsemf' from OTEL_METRICS_EXPORTER if present to prevent validation errors
+    from _import_exporters in OTel dependencies which would try to load exporters
+    We will contribute emf exporter to upstream for supporting OTel metrics in SDK
+
+    Returns:
+    bool: True if "awsemf" was found and removed, False otherwise.
+    """
+    # Get the current exporter value
+    exporter_value = os.environ.get("OTEL_METRICS_EXPORTER", "")
+
+    # Check if it's empty
+    if not exporter_value:
+        return False
+
+    # Split by comma and convert to list
+    exporters = [exp.strip() for exp in exporter_value.split(",")]
+
+    # Check if awsemf is in the list
+    if "awsemf" not in exporters:
+        return False
+
+    # Remove awsemf from the list
+    exporters.remove("awsemf")
+
+    # Join the remaining exporters and update the environment variable
+    new_value = ",".join(exporters) if exporters else ""
+
+    # Set the new value (or unset if empty)
+    if new_value:
+        os.environ["OTEL_METRICS_EXPORTER"] = new_value
+    elif "OTEL_METRICS_EXPORTER" in os.environ:
+        del os.environ["OTEL_METRICS_EXPORTER"]
+
+    return True
+
+
+def create_emf_exporter():
+    """Create and configure the CloudWatch EMF exporter."""
+    try:
+        # Check if botocore is available before importing the EMF exporter
+        if not is_installed("botocore"):
+            _logger.warning("botocore is not installed. EMF exporter requires botocore")
+            return None
+
+        # pylint: disable=import-outside-toplevel
+        from amazon.opentelemetry.distro.exporter.aws.metrics.aws_cloudwatch_emf_exporter import (
+            AwsCloudWatchEmfExporter,
+        )
+
+        log_header_setting = _validate_and_fetch_logs_header()
+
+        if not log_header_setting.is_valid:
+            return None
+
+        return AwsCloudWatchEmfExporter(
+            namespace=log_header_setting.namespace,
+            log_group_name=log_header_setting.log_group,
+            log_stream_name=log_header_setting.log_stream,
+        )
+    # pylint: disable=broad-exception-caught
+    except Exception as errors:
+        _logger.error("Failed to create EMF exporter: %s", errors)
+        return None

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py

@@ -4,6 +4,19 @@
 import sys
 from logging import Logger, getLogger
 
+from amazon.opentelemetry.distro._utils import get_aws_region, is_agent_observability_enabled
+from amazon.opentelemetry.distro.aws_opentelemetry_configurator import (
+    APPLICATION_SIGNALS_ENABLED_CONFIG,
+    OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
+    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+    OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT,
+    OTEL_LOGS_EXPORTER,
+    OTEL_METRICS_EXPORTER,
+    OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
+    OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED,
+    OTEL_TRACES_EXPORTER,
+    OTEL_TRACES_SAMPLER,
+)
 from amazon.opentelemetry.distro.patches._instrumentation_patch import apply_instrumentation_patches
 from opentelemetry.distro import OpenTelemetryDistro
 from opentelemetry.environment_variables import OTEL_PROPAGATORS, OTEL_PYTHON_ID_GENERATOR
@@ -65,5 +78,44 @@ def _configure(self, **kwargs):
             OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, "base2_exponential_bucket_histogram"
         )
 
+        if is_agent_observability_enabled():
+            # "otlp" is already native OTel default, but we set them here to be explicit
+            # about intended configuration for agent observability
+            os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp")
+            os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp")
+            os.environ.setdefault(OTEL_METRICS_EXPORTER, "awsemf")
+
+            # Set GenAI capture content default
+            os.environ.setdefault(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "true")
+
+            # Set OTLP endpoints with AWS region if not already set
+            region = get_aws_region()
+            if region:
+                os.environ.setdefault(
+                    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, f"https://xray.{region}.amazonaws.com/v1/traces"
+                )
+                os.environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, f"https://logs.{region}.amazonaws.com/v1/logs")
+            else:
+                _logger.warning(
+                    "AWS region could not be determined. OTLP endpoints will not be automatically configured. "
+                    "Please set AWS_REGION environment variable or configure OTLP endpoints manually."
+                )
+
+            # Set sampler default
+            os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on")
+
+            # Set disabled instrumentations default
+            os.environ.setdefault(
+                OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
+                "http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
+                "botocore,boto3,urllib3,requests,starlette",
+            )
+
+            # Set logging auto instrumentation default
+            os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
+
+            # Disable AWS Application Signals by default
+            os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false")
+
         if kwargs.get("apply_patches", True):
             apply_instrumentation_patches()