Merge branch 'main' into setup-unsampled-span-pipeline

Author: yiyuan-he

.github/workflows/release-lambda.yml

@@ -12,7 +12,7 @@ on:
         default: 'us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, af-south-1, ap-east-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, il-central-1, me-central-1, me-south-1'
 
 env:
-  COMMERCIAL_REGIONS: us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1
+  COMMERCIAL_REGIONS: us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-3, ap-northeast-2, ap-southeast-1, ap-southeast-2, ap-northeast-1, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1, ap-southeast-5, ap-southeast-7, mx-central-1, ca-west-1, cn-north-1, cn-northwest-1
   LAYER_NAME: AWSOpenTelemetryDistroPython
 
 permissions:

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -35,3 +35,32 @@ def is_installed(req: str) -> bool:
 def is_agent_observability_enabled() -> bool:
     """Is the Agentic AI monitoring flag set to true?"""
     return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"
+
+
+def get_aws_region() -> str:
+    """Get AWS region using botocore session.
+
+    botocore automatically checks in the following priority order:
+    1. AWS_REGION environment variable
+    2. AWS_DEFAULT_REGION environment variable
+    3. AWS CLI config file (~/.aws/config)
+    4. EC2 instance metadata service
+
+    Returns:
+        The AWS region if found, None otherwise.
+    """
+    if is_installed("botocore"):
+        try:
+            from botocore import session  # pylint: disable=import-outside-toplevel
+
+            botocore_session = session.Session()
+            if botocore_session.region_name:
+                return botocore_session.region_name
+        except (ImportError, AttributeError):
+            # botocore failed to determine region
+            pass
+
+    _logger.warning(
+        "AWS region not found. Please set AWS_REGION environment variable or configure AWS CLI with 'aws configure'."
+    )
+    return None

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -102,6 +102,14 @@
 # UDP package size is not larger than 64KB
 LAMBDA_SPAN_EXPORT_BATCH_SIZE = 10
 
+OTEL_TRACES_EXPORTER = "OTEL_TRACES_EXPORTER"
+OTEL_LOGS_EXPORTER = "OTEL_LOGS_EXPORTER"
+OTEL_METRICS_EXPORTER = "OTEL_METRICS_EXPORTER"
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+OTEL_TRACES_SAMPLER = "OTEL_TRACES_SAMPLER"
+OTEL_PYTHON_DISABLED_INSTRUMENTATIONS = "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS"
+OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED = "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"
+
 _logger: Logger = getLogger(__name__)
 
 

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py

@@ -4,6 +4,19 @@
 import sys
 from logging import Logger, getLogger
 
+from amazon.opentelemetry.distro._utils import get_aws_region, is_agent_observability_enabled
+from amazon.opentelemetry.distro.aws_opentelemetry_configurator import (
+    APPLICATION_SIGNALS_ENABLED_CONFIG,
+    OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
+    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+    OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT,
+    OTEL_LOGS_EXPORTER,
+    OTEL_METRICS_EXPORTER,
+    OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
+    OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED,
+    OTEL_TRACES_EXPORTER,
+    OTEL_TRACES_SAMPLER,
+)
 from amazon.opentelemetry.distro.patches._instrumentation_patch import apply_instrumentation_patches
 from opentelemetry.distro import OpenTelemetryDistro
 from opentelemetry.environment_variables import OTEL_PROPAGATORS, OTEL_PYTHON_ID_GENERATOR
@@ -65,5 +78,44 @@ def _configure(self, **kwargs):
             OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, "base2_exponential_bucket_histogram"
         )
 
+        if is_agent_observability_enabled():
+            # "otlp" is already native OTel default, but we set them here to be explicit
+            # about intended configuration for agent observability
+            os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp")
+            os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp")
+            os.environ.setdefault(OTEL_METRICS_EXPORTER, "awsemf")
+
+            # Set GenAI capture content default
+            os.environ.setdefault(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "true")
+
+            # Set OTLP endpoints with AWS region if not already set
+            region = get_aws_region()
+            if region:
+                os.environ.setdefault(
+                    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, f"https://xray.{region}.amazonaws.com/v1/traces"
+                )
+                os.environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, f"https://logs.{region}.amazonaws.com/v1/logs")
+            else:
+                _logger.warning(
+                    "AWS region could not be determined. OTLP endpoints will not be automatically configured. "
+                    "Please set AWS_REGION environment variable or configure OTLP endpoints manually."
+                )
+
+            # Set sampler default
+            os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on")
+
+            # Set disabled instrumentations default
+            os.environ.setdefault(
+                OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
+                "http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
+                "botocore,boto3,urllib3,requests,starlette",
+            )
+
+            # Set logging auto instrumentation default
+            os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
+
+            # Disable AWS Application Signals by default
+            os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false")
+
         if kwargs.get("apply_patches", True):
             apply_instrumentation_patches()

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py

@@ -1,12 +1,20 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from typing import Dict, Optional
+import logging
+from typing import Dict, Optional, Sequence
 
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
 from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
+from amazon.opentelemetry.distro.llo_handler import LLOHandler
+from opentelemetry._logs import get_logger_provider
 from opentelemetry.exporter.otlp.proto.http import Compression
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExportResult
+
+logger = logging.getLogger(__name__)
 
 
 class OTLPAwsSpanExporter(OTLPSpanExporter):
@@ -23,6 +31,7 @@ def __init__(
     ):
         self._aws_region = None
         self._logger_provider = logger_provider
+        self._llo_handler = None
 
         if endpoint:
             self._aws_region = endpoint.split(".")[1]
@@ -38,3 +47,29 @@ def __init__(
             compression,
             session=AwsAuthSession(aws_region=self._aws_region, service="xray"),
         )
+
+    def _ensure_llo_handler(self):
+        """Lazily initialize LLO handler when needed to avoid initialization order issues"""
+        if self._llo_handler is None and is_agent_observability_enabled():
+            if self._logger_provider is None:
+                try:
+                    self._logger_provider = get_logger_provider()
+                except Exception as exc:  # pylint: disable=broad-exception-caught
+                    logger.debug("Failed to get logger provider: %s", exc)
+                    return False
+
+            if self._logger_provider:
+                self._llo_handler = LLOHandler(self._logger_provider)
+                return True
+
+        return self._llo_handler is not None
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        try:
+            if is_agent_observability_enabled() and self._ensure_llo_handler():
+                llo_processed_spans = self._llo_handler.process_spans(spans)
+                return super().export(llo_processed_spans)
+        except Exception:  # pylint: disable=broad-exception-caught
+            return SpanExportResult.FAILURE
+
+        return super().export(spans)

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/llo_handler.py

@@ -0,0 +1,32 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+from opentelemetry.sdk._logs import LoggerProvider
+
+
+class LLOHandler:
+    """
+    Utility class for handling Large Language Objects (LLO) in OpenTelemetry spans.
+
+    LLOHandler performs three primary functions:
+    1. Identifies input/output prompt content in spans
+    2. Extracts and transforms these attributes into an OpenTelemetry Gen AI Event
+    3. Filters input/output prompts from spans to maintain privacy and reduce span size
+
+    This LLOHandler supports the following third-party instrumentation libraries:
+    - Strands
+    - OpenInference
+    - Traceloop/OpenLLMetry
+    - OpenLIT
+    """
+
+    def __init__(self, logger_provider: LoggerProvider):
+        """
+        Initialize an LLOHandler with the specified logger provider.
+
+        This constructor sets up the event logger provider, configures the event logger,
+        and initializes the patterns used to identify LLO attributes.
+
+        Args:
+            logger_provider: The OpenTelemetry LoggerProvider used for emitting events.
+                           Global LoggerProvider instance injected from our AwsOpenTelemetryConfigurator
+        """