[Python] SigV4 logs release testing (#407)

*Issue description:*
Need to implement and validate SigV4 log functionality for Python EC2
ADOT (Stand-Alone ADOT) use case. The primary focus is on validating
logs sent to the otlp_sigv4_logs log group using a consistent and
reliable approach that can be replicated across different language
implementations.

*Description of changes:*
1. Updated Terraform configuration:
   - Added OTLP logs exporter configuration
   - Set log group to otlp_sigv4_logs
   - Enabled Python logging auto-instrumentation
   - Updated environment variables for SigV4 authentication

2. Added validation resources:
   - Created new log.mustache template for SigV4 logs validation
   - Added log-validation.yml configuration
- Updated PredefinedExpectedTemplate.java to include SigV4 logs template
   - Modified CWLogValidator.java to support SigV4 logs validation

3. Workflow Updates:
   - Added new validation step for SigV4 logs
   - Configured TEST_LOG_GROUP_NAME environment variable

*Rollback procedure:*
1. Reverting the Terraform configuration changes
2. Removing the added validation files and code changes
3. Removing the SigV4 logs validation step from the workflow

Test run:
https://github.com/aws-observability/aws-application-signals-test-framework/actions/runs/15149888024/job/42593928321

<Can we safely revert this commit if needed? If not, detail what must be
done to safely revert and why it is needed.>

*Ensure you've run the following tests on your changes and include the
link below:*

To do so, create a `test.yml` file with `name: Test` and workflow
description to test your changes, then remove the file for your PR. Link
your test run in your PR description. This process is a short term
solution while we work on creating a staging environment for testing.

NOTE: TESTS RUNNING ON A SINGLE EKS CLUSTER CANNOT BE RUN IN PARALLEL.
See the
[needs](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idneeds)
keyword to run tests in succession.
- Run Java EKS on `e2e-playground` in us-east-1 and eu-central-2
- Run Python EKS on `e2e-playground` in us-east-1 and eu-central-2
- Run metric limiter on EKS cluster `e2e-playground` in us-east-1 and
eu-central-2
- Run EC2 tests in all regions
- Run K8s on a separate K8s cluster (check IAD test account for master
node endpoints; these will change as we create and destroy clusters for
OS patching)

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Jeel Mehta <[email protected]>

Author: Jeel-mehta

.github/workflows/python-ec2-adot-sigv4-test.yml

@@ -40,6 +40,7 @@ env:
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
   METRIC_NAMESPACE: ApplicationSignals
   LOG_GROUP_NAME: aws/spans
+  TEST_LOG_GROUP_NAME: otlp_logs
   TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE}
 
 jobs:
@@ -121,6 +122,7 @@ jobs:
               -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \
               -var="language_version=${{ env.PYTHON_VERSION }}" \
               -var="cpu_architecture=${{ env.CPU_ARCHITECTURE }}" \
+              -var="test_log_group=${{ env.TEST_LOG_GROUP_NAME }}" \
             || deployment_failed=$?
           
             if [ $deployment_failed -eq 1 ]; then
@@ -170,7 +172,7 @@ jobs:
       # Validation for pulse telemetry data
       - name: Validate generated EMF logs
         id: log-validation
-        run: ./gradlew validator:run --args='-c python/ec2/adot-sigv4/log-validation.yml
+        run: ./gradlew validator:run --args='-c python/ec2/adot-aws-otlp/log-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://localhost:8000
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
@@ -187,7 +189,7 @@ jobs:
       - name: Validate generated metrics
         id: metric-validation
         if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
-        run: ./gradlew validator:run --args='-c python/ec2/adot-sigv4/metric-validation.yml
+        run: ./gradlew validator:run --args='-c python/ec2/adot-aws-otlp/metric-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://localhost:8000
           --remote-service-deployment-name python-sample-remote-application-${{ env.TESTING_ID }}
@@ -204,7 +206,7 @@ jobs:
       - name: Validate generated traces
         id: trace-validation
         if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
-        run: ./gradlew validator:run --args='-c python/ec2/adot-sigv4/trace-validation.yml
+        run: ./gradlew validator:run --args='-c python/ec2/adot-aws-otlp/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://localhost:8000
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
@@ -218,6 +220,23 @@ jobs:
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
           --rollup'
+      
+      - name: Validate generated otlp logs
+        id: log-validation-1
+        run: ./gradlew validator:run --args='-c python/ec2/adot-aws-otlp/logs/log-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
+          --endpoint http://localhost:8000
+          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
+          --region ${{ env.E2E_TEST_AWS_REGION }}
+          --account-id ${{ env.E2E_TEST_ACCOUNT_ID }}
+          --metric-namespace ${{ env.METRIC_NAMESPACE }}
+          --log-group ${{ env.TEST_LOG_GROUP_NAME }}
+          --service-name python-sample-application-${{ env.TESTING_ID }}
+          --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
+          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
+          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
+          --rollup'
 
       - name: Refresh AWS Credentials
         if: always()

terraform/python/ec2/adot-sigv4/main.tf

@@ -163,10 +163,13 @@ resource "null_resource" "main_service_setup" {
       export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
       export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=false
       export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
-      export OTEL_LOGS_EXPORT=none \
+      export OTEL_LOGS_EXPORTER=otlp \
       export OTEL_METRICS_EXPORTER=none \
-      export OTEL_TRACES_EXPORTER=otlp
+      export OTEL_TRACES_EXPORTER=otlp \
       export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://xray.${var.aws_region}.amazonaws.com/v1/traces \
+      export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://logs.${var.aws_region}.amazonaws.com/v1/logs \
+      export OTEL_EXPORTER_OTLP_LOGS_HEADERS=x-aws-log-group=${var.test_log_group},x-aws-log-stream=default \
+      export OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true \
       export OTEL_SERVICE_NAME=python-sample-application-${var.test_id}
       export OTEL_TRACES_SAMPLER=always_on
       python${var.language_version} manage.py migrate
@@ -284,10 +287,13 @@ resource "null_resource" "remote_service_setup" {
       export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
       export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=false
       export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
-      export OTEL_LOGS_EXPORT=none \
+      export OTEL_LOGS_EXPORTER=otlp \
       export OTEL_METRICS_EXPORTER=none \
-      export OTEL_TRACES_EXPORTER=otlp
+      export OTEL_TRACES_EXPORTER=otlp \
       export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://xray.${var.aws_region}.amazonaws.com/v1/traces \
+      export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://logs.${var.aws_region}.amazonaws.com/v1/logs \
+      export OTEL_EXPORTER_OTLP_LOGS_HEADERS=x-aws-log-group=${var.test_log_group},x-aws-log-stream=default \
+      export OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true \
       export OTEL_SERVICE_NAME=python-sample-remote-application-${var.test_id}
       export OTEL_TRACES_SAMPLER=always_on
       python${var.language_version} manage.py migrate

terraform/python/ec2/adot-sigv4/variables.tf

@@ -43,4 +43,8 @@ variable "language_version" {
 
 variable "cpu_architecture" {
   default = "x86_64"
+}
+
+variable "test_log_group" {
+  default = "otlp_logs"
 }

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -222,23 +222,26 @@ public enum PredefinedExpectedTemplate implements FileConfig {
 
   /** Python EC2 ADOT SigV4 (Stand Alone ADOT) Test Case Validations */
   PYTHON_EC2_ADOT_SIGV4_OUTGOING_HTTP_CALL_LOG(
-      "/expected-data-template/python/ec2/adot-sigv4/outgoing-http-call-log.mustache"),
+      "/expected-data-template/python/ec2/adot-aws-otlp/traces/outgoing-http-call-log.mustache"),
   PYTHON_EC2_ADOT_SIGV4_OUTGOING_HTTP_CALL_METRIC(
-      "/expected-data-template/python/ec2/adot-sigv4/outgoing-http-call-metric.mustache"),
+      "/expected-data-template/python/ec2/adot-aws-otlp/traces/outgoing-http-call-metric.mustache"),
   PYTHON_EC2_ADOT_SIGV4_OUTGOING_HTTP_CALL_TRACE(
-      "/expected-data-template/python/ec2/adot-sigv4/outgoing-http-call-trace.mustache"),
+      "/expected-data-template/python/ec2/adot-aws-otlp/traces/outgoing-http-call-trace.mustache"),
 
-  PYTHON_EC2_ADOT_SIGV4_AWS_SDK_CALL_LOG("/expected-data-template/python/ec2/adot-sigv4/aws-sdk-call-log.mustache"),
-  PYTHON_EC2_ADOT_SIGV4_AWS_SDK_CALL_METRIC("/expected-data-template/python/ec2/adot-sigv4/aws-sdk-call-metric.mustache"),
-  PYTHON_EC2_ADOT_SIGV4_AWS_SDK_CALL_TRACE("/expected-data-template/python/ec2/adot-sigv4/aws-sdk-call-trace.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_AWS_SDK_CALL_LOG("/expected-data-template/python/ec2/adot-aws-otlp/traces/aws-sdk-call-log.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_AWS_SDK_CALL_METRIC("/expected-data-template/python/ec2/adot-aws-otlp/traces/aws-sdk-call-metric.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_AWS_SDK_CALL_TRACE("/expected-data-template/python/ec2/adot-aws-otlp/traces/aws-sdk-call-trace.mustache"),
 
-  PYTHON_EC2_ADOT_SIGV4_REMOTE_SERVICE_LOG("/expected-data-template/python/ec2/adot-sigv4/remote-service-log.mustache"),
-  PYTHON_EC2_ADOT_SIGV4_REMOTE_SERVICE_METRIC("/expected-data-template/python/ec2/adot-sigv4/remote-service-metric.mustache"),
-  PYTHON_EC2_ADOT_SIGV4_REMOTE_SERVICE_TRACE("/expected-data-template/python/ec2/adot-sigv4/remote-service-trace.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_REMOTE_SERVICE_LOG("/expected-data-template/python/ec2/adot-aws-otlp/traces/remote-service-log.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_REMOTE_SERVICE_METRIC("/expected-data-template/python/ec2/adot-aws-otlp/traces/remote-service-metric.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_REMOTE_SERVICE_TRACE("/expected-data-template/python/ec2/adot-aws-otlp/traces/remote-service-trace.mustache"),
 
-  PYTHON_EC2_ADOT_SIGV4_CLIENT_CALL_LOG("/expected-data-template/python/ec2/adot-sigv4/client-call-log.mustache"),
-  PYTHON_EC2_ADOT_SIGV4_CLIENT_CALL_METRIC("/expected-data-template/python/ec2/adot-sigv4/client-call-metric.mustache"),
-  PYTHON_EC2_ADOT_SIGV4_CLIENT_CALL_TRACE("/expected-data-template/python/ec2/adot-sigv4/client-call-trace.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_CLIENT_CALL_LOG("/expected-data-template/python/ec2/adot-aws-otlp/traces/client-call-log.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_CLIENT_CALL_METRIC("/expected-data-template/python/ec2/adot-aws-otlp/traces/client-call-metric.mustache"),
+  PYTHON_EC2_ADOT_SIGV4_CLIENT_CALL_TRACE("/expected-data-template/python/ec2/adot-aws-otlp/traces/client-call-trace.mustache"),
+
+  /** Python EC2 ADOT SigV4 Log Exporter Test Case Validation */
+  PYTHON_EC2_ADOT_OTLP_LOG("/expected-data-template/python/ec2/adot-aws-otlp/logs/log.mustache"),
 
   /** Python K8S Test Case Validations */
   PYTHON_K8S_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/k8s/outgoing-http-call-log.mustache"),

validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java

@@ -65,32 +65,36 @@ public void validate() throws Exception {
           // which are in normal text as they are needed for
           // the filter expressions for retrieving the actual logs.
           log.info("Searching for expected log: {}", expectedAttributes);
-          String operation = (String) expectedAttributes.get("Operation");
-          String remoteService = (String) expectedAttributes.get("RemoteService");
-          String remoteOperation = (String) expectedAttributes.get("RemoteOperation");
-          String remoteResourceType = (String) expectedAttributes.get("RemoteResourceType");
-          String remoteResourceIdentifier = (String) expectedAttributes.get("RemoteResourceIdentifier");
-
           Map<String, Object> actualLog;
 
-          // Parsing unique identifiers in OTLP spans
-          if (operation == null) {
-            operation = (String) expectedAttributes.get("attributes[\\\"aws.local.operation\\\"]");
-            remoteService = (String) expectedAttributes.get("attributes[\\\"aws.remote.service\\\"]");
-            remoteOperation = (String) expectedAttributes.get("attributes[\\\"aws.remote.operation\\\"]");
-            // Runtime metrics have no operation at all, we must ensure we are in the proper use case
-            if (operation != null) {
-              actualLog = this.getActualOtelSpanLog(operation, remoteService, remoteOperation);
-            } else {
-              // No operation at all -> Runtime metric
+          if (isAwsOtlpLog(expectedAttributes)) {
+                    actualLog = this.getActualAwsOtlpLog();
+          } else {
+            String operation = (String) expectedAttributes.get("Operation");
+            String remoteService = (String) expectedAttributes.get("RemoteService");
+            String remoteOperation = (String) expectedAttributes.get("RemoteOperation");
+            String remoteResourceType = (String) expectedAttributes.get("RemoteResourceType");
+            String remoteResourceIdentifier = (String) expectedAttributes.get("RemoteResourceIdentifier");
+
+           // Parsing unique identifiers in OTLP spans
+            if (operation == null) {
+              operation = (String) expectedAttributes.get("attributes[\\\"aws.local.operation\\\"]");
+              remoteService = (String) expectedAttributes.get("attributes[\\\"aws.remote.service\\\"]");
+              remoteOperation = (String) expectedAttributes.get("attributes[\\\"aws.remote.operation\\\"]");
+              // Runtime metrics have no operation at all, we must ensure we are in the proper use case
+              if (operation != null) {
+                actualLog = this.getActualOtelSpanLog(operation, remoteService, remoteOperation);
+              } else {
+                // No operation at all -> Runtime metric
+                actualLog =
+                  this.getActualLog(operation, remoteService, remoteOperation, remoteResourceType, remoteResourceIdentifier);
+              }
+            }
+            else {
               actualLog =
                 this.getActualLog(operation, remoteService, remoteOperation, remoteResourceType, remoteResourceIdentifier);
             }
           }
-          else {
-            actualLog =
-              this.getActualLog(operation, remoteService, remoteOperation, remoteResourceType, remoteResourceIdentifier);
-          }
           log.info("Value of an actual log: {}", actualLog);
 
           if (actualLog == null) throw new BaseException(ExceptionCode.EXPECTED_LOG_NOT_FOUND);
@@ -147,6 +151,13 @@ private JsonifyArrayList<Map<String, Object>> getExpectedAttributes() throws Exc
     return flattenedJsonMapForExpectedLogArray;
   }
 
+  private boolean isAwsOtlpLog(Map<String, Object> expectedAttributes) {
+    // OTLP SigV4 logs have 'body' as a top-level attribute
+    return expectedAttributes.containsKey("body") &&
+           expectedAttributes.containsKey("severityNumber") &&
+           expectedAttributes.containsKey("severityText");
+  }
+
   private Map<String, Object> getActualLog(
     String operation, String remoteService, String remoteOperation, String remoteResourceType, String remoteResourceIdentifier) throws Exception {
     String dependencyFilter = null;
@@ -223,6 +234,27 @@ private Map<String, Object> getActualOtelSpanLog(String operation, String remote
     return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage());
   }
 
+  private Map<String, Object> getActualAwsOtlpLog() throws Exception {
+    String filterPattern= String.format(
+            "{ ($.attributes.otelServiceName = \"%s\") && ($.body = \"This is a custom log for validation testing\") }",
+            context.getServiceName()
+        );
+    log.info("Filter Pattern for OTLP Log Search: " + filterPattern);
+
+    List<FilteredLogEvent> retrievedLogs =
+        this.cloudWatchService.filterLogs(
+            context.getLogGroup(),
+            filterPattern,
+            System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5),
+            10);
+
+    if (retrievedLogs == null || retrievedLogs.isEmpty()) {
+        throw new BaseException(ExceptionCode.EMPTY_LIST);
+    }
+
+    return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage());
+  }
+
   @Override
   public void init(
       Context context,

validator/src/main/resources/expected-data-template/python/ec2/adot-aws-otlp/logs/log.mustache

@@ -0,0 +1,20 @@
+[{
+  "resource": {
+    "attributes": {
+      "aws.local.service": "{{serviceName}}",
+      "service.name": "{{serviceName}}",
+      "cloud.provider": "aws",
+      "cloud.region": "{{region}}",
+      "cloud.account.id": "{{accountId}}",
+      "cloud.platform": "aws_ec2"
+    }
+  },
+  "severityNumber": "^[0-9]+$",
+  "severityText": "{{severityText}}",
+  "body": "This is a custom log for validation testing",
+  "traceId": "{{traceId}}",
+  "spanId": "{{spanId}}",
+  "attributes": {
+    "otelServiceName": "{{serviceName}}"
+  }
+}]