Merge branch 'main' into pr1-emf-exporter-foundation

Author: mxiamxia

.github/workflows/application-signals-e2e-test.yml

@@ -46,17 +46,6 @@ jobs:
   # DEFAULT SETTING: {Python Version}, EC2, AMD64, AL2
   #
 
-  default-py38-amd64:
-    needs: [ upload-main-build ]
-    uses: aws-observability/aws-application-signals-test-framework/.github/workflows/python-ec2-default-test.yml@main
-    secrets: inherit
-    with:
-      aws-region: us-east-1
-      caller-workflow-name: 'main-build'
-      python-version: '3.8'
-      cpu-architecture: 'x86_64'
-      staging-wheel-name: ${{ inputs.staging-wheel-name }}
-
   default-py39-amd64:
     needs: [ upload-main-build ]
     uses: aws-observability/aws-application-signals-test-framework/.github/workflows/python-ec2-default-test.yml@main
@@ -106,19 +95,8 @@ jobs:
   # DEFAULT SETTING: {Python Version}, EKS, AMD64, AL2
   #
 
-  eks-py38-amd64:
-    uses: aws-observability/aws-application-signals-test-framework/.github/workflows/python-eks-test.yml@main
-    secrets: inherit
-    with:
-      aws-region: us-east-1
-      test-cluster-name: 'e2e-python-adot-test'
-      adot-image-name: ${{ inputs.adot-image-name }}
-      caller-workflow-name: 'main-build'
-      python-version: '3.8'
-
   eks-py39-amd64:
     if: ${{ always() }}
-    needs: eks-py38-amd64
     uses: aws-observability/aws-application-signals-test-framework/.github/workflows/python-eks-test.yml@main
     secrets: inherit
     with:

.github/workflows/pr_build.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false # ensures the entire test matrix is run, even if one permutation fails
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Checkout Repo @ SHA - ${{ github.sha }}
         uses: actions/checkout@v4

Dockerfile

@@ -18,22 +18,10 @@ ADD aws-opentelemetry-distro/ ./aws-opentelemetry-distro/
 # * https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/azure-functions/recover-python-functions.md#troubleshoot-cannot-import-cygrpc
 RUN sed -i "/opentelemetry-exporter-otlp-proto-grpc/d" ./aws-opentelemetry-distro/pyproject.toml
 
-# urllib3 recently made a release that drops support for Python 3.8
-# Our sdk depends on botocore which pulls in the version of urllib3 based Python runtime it detects.
-# The rule is that if current pip command version is > 3.9 botocore will pick up the latest urllib3,
-# otherwise it picks up the older urllib3 that is compatible with Python 3.8.
-# https://github.com/boto/botocore/blob/develop/requirements-docs.txt
-# Since this Dockerfile currently uses the fixed Python 3.11 base image to pull the required dependencies,
-# EKS and ECS applications will encounter a runtime error for Python 3.8 compatibility.
-# Our fix is to temporarily restrict the urllib3 version to one that works for all supported Python versions 
-# that we currently commit to support (notably 3.8). 
-# We also pin the setuptools version for similar issues with the library dropping 3.8 support.
-# https://github.com/pypa/setuptools/blame/main/pkg_resources/__init__.py#L24
-# TODO: Remove these temporary workarounds once we deprecate Python 3.8 support since it has reached end-of-life.
-RUN mkdir workspace && pip install setuptools==75.2.0 urllib3==2.2.3 --target workspace ./aws-opentelemetry-distro
+RUN mkdir workspace && pip install --target workspace ./aws-opentelemetry-distro
 
 # Stage 2: Build the cp-utility binary
-FROM public.ecr.aws/docker/library/rust:1.82 as builder
+FROM public.ecr.aws/docker/library/rust:1.87 as builder
 
 WORKDIR /usr/src/cp-utility
 COPY ./tools/cp-utility .

README.md

@@ -35,7 +35,7 @@ This project is licensed under the Apache-2.0 License.
 ## Notices
 
 ### Python Version Support
-This project ensures compatibility with the following supported Python versions: 3.8, 3.9, 3.10, 3.11, 3.12
+This project ensures compatibility with the following supported Python versions: 3.9, 3.10, 3.11, 3.12
 
 ### Note on Amazon CloudWatch Application Signals
 [Amazon CloudWatch Application Signals](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Application-Monitoring-Sections.html) components are designed to seamlessly work with all library instrumentations offered by [OpenTelemetry Python auto-instrumentation](https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/instrumentation/README.md). While upstream OpenTelemetry Python instrumentations are in beta, Application Signals components are stable, production ready and have also been tested for popular libraries/frameworks such as [Django, Boto3, and others](https://github.com/aws-observability/aws-otel-python-instrumentation/tree/main/contract-tests/images/applications). We will prioritize backward compatibility for Application Signals components, striving to ensure that they remain functional even in the face of potential breaking changes introduced by OpenTelemetry upstream libraries. Please [raise an issue](https://github.com/aws-observability/aws-otel-python-instrumentation/blob/main/CONTRIBUTING.md#reporting-bugsfeature-requests) if you notice Application Signals doesn't work for a particular OpenTelemetry supported library.

aws-opentelemetry-distro/pyproject.toml

@@ -8,7 +8,7 @@ dynamic = ["version"]
 description = "AWS OpenTelemetry Python Distro"
 readme = "README.rst"
 license = "Apache-2.0"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
   { name = "Amazon Web Services" },
 ]
@@ -18,7 +18,6 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -1,13 +1,16 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import os
 import sys
 from logging import Logger, getLogger
 
 import pkg_resources
 
 _logger: Logger = getLogger(__name__)
 
+AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
+
 
 def is_installed(req: str) -> bool:
     """Is the given required package installed?"""
@@ -21,3 +24,8 @@ def is_installed(req: str) -> bool:
         _logger.debug("Skipping instrumentation patch: package %s, exception: %s", req, exc)
         return False
     return True
+
+
+def is_agent_observability_enabled() -> bool:
+    """Is the Agentic AI monitoring flag set to true?"""
+    return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -11,6 +11,7 @@
 
 from amazon.opentelemetry.distro._aws_attribute_keys import AWS_LOCAL_SERVICE
 from amazon.opentelemetry.distro._aws_resource_attribute_configurator import get_service_attribute
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
 from amazon.opentelemetry.distro.always_record_sampler import AlwaysRecordSampler
 from amazon.opentelemetry.distro.attribute_propagating_span_processor_builder import (
     AttributePropagatingSpanProcessorBuilder,
@@ -27,7 +28,7 @@
 from amazon.opentelemetry.distro.sampler.aws_xray_remote_sampler import AwsXRayRemoteSampler
 from amazon.opentelemetry.distro.scope_based_exporter import ScopeBasedPeriodicExportingMetricReader
 from amazon.opentelemetry.distro.scope_based_filtering_view import ScopeBasedRetainingView
-from opentelemetry._logs import set_logger_provider
+from opentelemetry._logs import get_logger_provider, set_logger_provider
 from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
 from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPHttpOTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
@@ -359,7 +360,15 @@ def _customize_span_exporter(span_exporter: SpanExporter, resource: Resource) ->
         _logger.info("Detected using AWS OTLP Traces Endpoint.")
 
         if isinstance(span_exporter, OTLPSpanExporter):
-            span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint)
+            if is_agent_observability_enabled():
+                # Span exporter needs an instance of logger provider in ai agent
+                # observability case because we need to split input/output prompts
+                # from span attributes and send them to the logs pipeline per
+                # the new Gen AI semantic convention from OTel
+                # ref: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
+                span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint, logger_provider=get_logger_provider())
+            else:
+                span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint)
 
         else:
             _logger.warning(

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py

@@ -6,6 +6,7 @@
 from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
 from opentelemetry.exporter.otlp.proto.http import Compression
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk._logs import LoggerProvider
 
 
 class OTLPAwsSpanExporter(OTLPSpanExporter):
@@ -18,8 +19,10 @@ def __init__(
         headers: Optional[Dict[str, str]] = None,
         timeout: Optional[int] = None,
         compression: Optional[Compression] = None,
+        logger_provider: Optional[LoggerProvider] = None,
     ):
         self._aws_region = None
+        self._logger_provider = logger_provider
 
         if endpoint:
             self._aws_region = endpoint.split(".")[1]

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py

@@ -311,6 +311,35 @@ def test_customize_span_exporter(self):
         self.assertIsInstance(customized_exporter._delegate, OTLPUdpSpanExporter)
         os.environ.pop("AWS_LAMBDA_FUNCTION_NAME", None)
 
+    def test_customize_span_exporter_with_agent_observability(self):
+        # Test that logger_provider is passed when agent observability is enabled
+        os.environ["AGENT_OBSERVABILITY_ENABLED"] = "true"
+        os.environ[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT] = "https://xray.us-east-1.amazonaws.com/v1/traces"
+
+        mock_logger_provider = MagicMock()
+        with patch(
+            "amazon.opentelemetry.distro.aws_opentelemetry_configurator.get_logger_provider",
+            return_value=mock_logger_provider,
+        ):
+            mock_exporter = MagicMock(spec=OTLPSpanExporter)
+            result = _customize_span_exporter(mock_exporter, Resource.get_empty())
+
+            self.assertIsInstance(result, OTLPAwsSpanExporter)
+            self.assertEqual(result._logger_provider, mock_logger_provider)
+
+        # Test that logger_provider is not passed when agent observability is disabled
+        os.environ["AGENT_OBSERVABILITY_ENABLED"] = "false"
+
+        mock_exporter = MagicMock(spec=OTLPSpanExporter)
+        result = _customize_span_exporter(mock_exporter, Resource.get_empty())
+
+        self.assertIsInstance(result, OTLPAwsSpanExporter)
+        self.assertIsNone(result._logger_provider)
+
+        # Clean up
+        os.environ.pop("AGENT_OBSERVABILITY_ENABLED", None)
+        os.environ.pop(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, None)
+
     def test_customize_span_exporter_sigv4(self):
 
         traces_good_endpoints = [

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_otlp_aws_span_exporter.py

@@ -0,0 +1,29 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from unittest import TestCase
+from unittest.mock import MagicMock
+
+from amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter import OTLPAwsSpanExporter
+from opentelemetry.sdk._logs import LoggerProvider
+
+
+class TestOTLPAwsSpanExporter(TestCase):
+    def test_init_with_logger_provider(self):
+        # Test initialization with logger_provider
+        mock_logger_provider = MagicMock(spec=LoggerProvider)
+        endpoint = "https://xray.us-east-1.amazonaws.com/v1/traces"
+
+        exporter = OTLPAwsSpanExporter(endpoint=endpoint, logger_provider=mock_logger_provider)
+
+        self.assertEqual(exporter._logger_provider, mock_logger_provider)
+        self.assertEqual(exporter._aws_region, "us-east-1")
+
+    def test_init_without_logger_provider(self):
+        # Test initialization without logger_provider (default behavior)
+        endpoint = "https://xray.us-west-2.amazonaws.com/v1/traces"
+
+        exporter = OTLPAwsSpanExporter(endpoint=endpoint)
+
+        self.assertIsNone(exporter._logger_provider)
+        self.assertEqual(exporter._aws_region, "us-west-2")