Merge branch 'main' into add-lambda-layer-default-region

Author: harrryr

.github/workflows/release-lambda.yml

@@ -12,7 +12,7 @@ on:
         default: 'us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, af-south-1, ap-east-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, il-central-1, me-central-1, me-south-1, ap-southeast-5, ap-southeast-7, mx-central-1, ca-west-1, cn-north-1, cn-northwest-1'
 
 env:
-  COMMERCIAL_REGIONS: us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1
+  COMMERCIAL_REGIONS: us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, ap-southeast-5, ap-southeast-7, mx-central-1, ca-west-1, cn-north-1, cn-northwest-1
   LAYER_NAME: AWSOpenTelemetryDistroPython
 
 permissions:

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -35,3 +35,32 @@ def is_installed(req: str) -> bool:
 def is_agent_observability_enabled() -> bool:
     """Is the Agentic AI monitoring flag set to true?"""
     return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"
+
+
+def get_aws_region() -> str:
+    """Get AWS region using botocore session.
+
+    botocore automatically checks in the following priority order:
+    1. AWS_REGION environment variable
+    2. AWS_DEFAULT_REGION environment variable
+    3. AWS CLI config file (~/.aws/config)
+    4. EC2 instance metadata service
+
+    Returns:
+        The AWS region if found, None otherwise.
+    """
+    if is_installed("botocore"):
+        try:
+            from botocore import session  # pylint: disable=import-outside-toplevel
+
+            botocore_session = session.Session()
+            if botocore_session.region_name:
+                return botocore_session.region_name
+        except (ImportError, AttributeError):
+            # botocore failed to determine region
+            pass
+
+    _logger.warning(
+        "AWS region not found. Please set AWS_REGION environment variable or configure AWS CLI with 'aws configure'."
+    )
+    return None

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -102,6 +102,14 @@
 # UDP package size is not larger than 64KB
 LAMBDA_SPAN_EXPORT_BATCH_SIZE = 10
 
+OTEL_TRACES_EXPORTER = "OTEL_TRACES_EXPORTER"
+OTEL_LOGS_EXPORTER = "OTEL_LOGS_EXPORTER"
+OTEL_METRICS_EXPORTER = "OTEL_METRICS_EXPORTER"
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+OTEL_TRACES_SAMPLER = "OTEL_TRACES_SAMPLER"
+OTEL_PYTHON_DISABLED_INSTRUMENTATIONS = "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS"
+OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED = "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"
+
 _logger: Logger = getLogger(__name__)
 
 
@@ -265,6 +273,17 @@ def _export_unsampled_span_for_lambda(trace_provider: TracerProvider, resource:
     )
 
 
+def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvider, resource: Resource = None):
+    if not is_agent_observability_enabled():
+        return
+
+    traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT)
+
+    span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint, logger_provider=get_logger_provider())
+
+    trace_provider.add_span_processor(BatchUnsampledSpanProcessor(span_exporter=span_exporter))
+
+
 def _is_defer_to_workers_enabled():
     return os.environ.get(OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG, "false").strip().lower() == "true"
 
@@ -408,9 +427,14 @@ def _customize_span_processors(provider: TracerProvider, resource: Resource) ->
     if _is_lambda_environment():
         provider.add_span_processor(AwsLambdaSpanProcessor())
 
+    # We always send 100% spans to Genesis platform for agent observability because
+    # AI applications typically have low throughput traffic patterns and require
+    # comprehensive monitoring to catch subtle failure modes like hallucinations
+    # and quality degradation that sampling could miss.
     # Add session.id baggage attribute to span attributes to support AI Agent use cases
     # enabling session ID tracking in spans.
     if is_agent_observability_enabled():
+        _export_unsampled_span_for_agent_observability(provider, resource)
 
         def session_id_predicate(baggage_key: str) -> bool:
             return baggage_key == "session.id"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py

@@ -4,6 +4,19 @@
 import sys
 from logging import Logger, getLogger
 
+from amazon.opentelemetry.distro._utils import get_aws_region, is_agent_observability_enabled
+from amazon.opentelemetry.distro.aws_opentelemetry_configurator import (
+    APPLICATION_SIGNALS_ENABLED_CONFIG,
+    OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
+    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+    OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT,
+    OTEL_LOGS_EXPORTER,
+    OTEL_METRICS_EXPORTER,
+    OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
+    OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED,
+    OTEL_TRACES_EXPORTER,
+    OTEL_TRACES_SAMPLER,
+)
 from amazon.opentelemetry.distro.patches._instrumentation_patch import apply_instrumentation_patches
 from opentelemetry.distro import OpenTelemetryDistro
 from opentelemetry.environment_variables import OTEL_PROPAGATORS, OTEL_PYTHON_ID_GENERATOR
@@ -65,5 +78,44 @@ def _configure(self, **kwargs):
             OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, "base2_exponential_bucket_histogram"
         )
 
+        if is_agent_observability_enabled():
+            # "otlp" is already native OTel default, but we set them here to be explicit
+            # about intended configuration for agent observability
+            os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp")
+            os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp")
+            os.environ.setdefault(OTEL_METRICS_EXPORTER, "awsemf")
+
+            # Set GenAI capture content default
+            os.environ.setdefault(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "true")
+
+            # Set OTLP endpoints with AWS region if not already set
+            region = get_aws_region()
+            if region:
+                os.environ.setdefault(
+                    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, f"https://xray.{region}.amazonaws.com/v1/traces"
+                )
+                os.environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, f"https://logs.{region}.amazonaws.com/v1/logs")
+            else:
+                _logger.warning(
+                    "AWS region could not be determined. OTLP endpoints will not be automatically configured. "
+                    "Please set AWS_REGION environment variable or configure OTLP endpoints manually."
+                )
+
+            # Set sampler default
+            os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on")
+
+            # Set disabled instrumentations default
+            os.environ.setdefault(
+                OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
+                "http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
+                "botocore,boto3,urllib3,requests,starlette",
+            )
+
+            # Set logging auto instrumentation default
+            os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
+
+            # Disable AWS Application Signals by default
+            os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false")
+
         if kwargs.get("apply_patches", True):
             apply_instrumentation_patches()