[Node] SigV4 logs release testing

.github/workflows/node-ec2-adot-sigv4-test.yml

@@ -40,7 +40,8 @@ env:
   E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
   METRIC_NAMESPACE: ApplicationSignals
-  LOG_GROUP_NAME: aws/spans
+  SPANS_LOG_GROUP_NAME: aws/spans
+  APPLICATION_LOGS_LOG_GROUP_NAME: otlp_logs # This log group was created manually to store application logs
   TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE}
 
 jobs:
@@ -121,6 +122,7 @@ jobs:
               -var="get_adot_instrumentation_command=${{ env.GET_ADOT_INSTRUMENTATION_COMMAND }}" \
               -var="language_version=${{ env.NODE_VERSION }}" \
               -var="cpu_architecture=${{ env.CPU_ARCHITECTURE }}" \
+              -var="application_logs_log_group=${{ env.APPLICATION_LOGS_LOG_GROUP_NAME }}" \
             || deployment_failed=$?
 
             if [ $deployment_failed -eq 1 ]; then
@@ -171,14 +173,14 @@ jobs:
       # Validation for pulse telemetry data
       - name: Validate generated EMF logs
         id: log-validation
-        run: ./gradlew validator:run --args='-c node/ec2/adot-sigv4/log-validation.yml
+        run: ./gradlew validator:run --args='-c node/ec2/adot-aws-otlp/log-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
           --region ${{ env.E2E_TEST_AWS_REGION }}
           --account-id ${{ env.E2E_TEST_ACCOUNT_ID }}
           --metric-namespace ${{ env.METRIC_NAMESPACE }}
-          --log-group ${{ env.LOG_GROUP_NAME }}
+          --log-group ${{ env.SPANS_LOG_GROUP_NAME }}
           --service-name node-sample-application-${{ env.TESTING_ID }}
           --remote-service-name node-sample-remote-application-${{ env.TESTING_ID }}
           --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
@@ -189,14 +191,14 @@ jobs:
       - name: Validate generated metrics
         id: metric-validation
         if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
-        run: ./gradlew validator:run --args='-c node/ec2/adot-sigv4/metric-validation.yml
+        run: ./gradlew validator:run --args='-c node/ec2/adot-aws-otlp/metric-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
           --remote-service-deployment-name node-sample-remote-application-${{ env.TESTING_ID }}
           --region ${{ env.E2E_TEST_AWS_REGION }}
           --account-id ${{ env.E2E_TEST_ACCOUNT_ID }}
           --metric-namespace ${{ env.METRIC_NAMESPACE }}
-          --log-group ${{ env.LOG_GROUP_NAME }}
+          --log-group ${{ env.SPANS_LOG_GROUP_NAME }}
           --service-name node-sample-application-${{ env.TESTING_ID }}
           --remote-service-name node-sample-remote-application-${{ env.TESTING_ID }}
           --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
@@ -207,14 +209,31 @@ jobs:
       - name: Validate generated traces
         id: trace-validation
         if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
-        run: ./gradlew validator:run --args='-c node/ec2/adot-sigv4/trace-validation.yml
+        run: ./gradlew validator:run --args='-c node/ec2/adot-aws-otlp/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
           --region ${{ env.E2E_TEST_AWS_REGION }}
           --account-id ${{ env.E2E_TEST_ACCOUNT_ID }}
           --metric-namespace ${{ env.METRIC_NAMESPACE }}
-          --log-group ${{ env.LOG_GROUP_NAME }}
+          --log-group ${{ env.SPANS_LOG_GROUP_NAME }}
+          --service-name node-sample-application-${{ env.TESTING_ID }}
+          --remote-service-name node-sample-remote-application-${{ env.TESTING_ID }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
+          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
+          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
+          --rollup'
+
+      - name: Validate generated otlp logs
+        id: application-log-validation
+        run: ./gradlew validator:run --args='-c node/ec2/adot-aws-otlp/application-log-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
+          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
+          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
+          --region ${{ env.E2E_TEST_AWS_REGION }}
+          --account-id ${{ env.E2E_TEST_ACCOUNT_ID }}
+          --metric-namespace ${{ env.METRIC_NAMESPACE }}
+          --log-group ${{ env.APPLICATION_LOGS_LOG_GROUP_NAME }}
           --service-name node-sample-application-${{ env.TESTING_ID }}
           --remote-service-name node-sample-remote-application-${{ env.TESTING_ID }}
           --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}

terraform/node/ec2/adot-sigv4/main.tf

@@ -154,11 +154,14 @@ resource "null_resource" "main_service_setup" {
       # Note: We use OTEL_NODE_DISABLED_INSTRUMENTATIONS=fs,dns,express to avoid
       # having to validate around the telemetry generated for middleware
       tmux send-keys -t frontend 'export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=false' C-m
-      tmux send-keys -t frontend 'export OTEL_LOGS_EXPORTER=none' C-m
+      tmux send-keys -t frontend 'export OTEL_LOGS_EXPORTER=otlp' C-m
       tmux send-keys -t frontend 'export OTEL_METRICS_EXPORTER=none' C-m
       tmux send-keys -t frontend 'export OTEL_TRACES_EXPORTER=otlp' C-m
       tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf' C-m
+      tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/protobuf' C-m
       tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://xray.${var.aws_region}.amazonaws.com/v1/traces' C-m
+      tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://logs.${var.aws_region}.amazonaws.com/v1/logs' C-m
+      tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_LOGS_HEADERS=x-aws-log-group=${var.application_logs_log_group},x-aws-log-stream=default' C-m
       tmux send-keys -t frontend 'export OTEL_NODE_DISABLED_INSTRUMENTATIONS=fs,dns,express' C-m
       tmux send-keys -t frontend 'export OTEL_SERVICE_NAME=node-sample-application-${var.test_id}' C-m
       tmux send-keys -t frontend 'export OTEL_TRACES_SAMPLER=always_on' C-m
@@ -263,11 +266,14 @@ resource "null_resource" "remote_service_setup" {
       # Note: We use OTEL_NODE_DISABLED_INSTRUMENTATIONS=fs,dns,express to avoid
       # having to validate around the telemetry generated for middleware
       tmux send-keys -t remote 'export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=false' C-m
-      tmux send-keys -t remote 'export OTEL_LOGS_EXPORTER=none' C-m
+      tmux send-keys -t remote 'export OTEL_LOGS_EXPORTER=otlp' C-m
       tmux send-keys -t remote 'export OTEL_METRICS_EXPORTER=none' C-m
       tmux send-keys -t remote 'export OTEL_TRACES_EXPORTER=otlp' C-m
       tmux send-keys -t remote 'export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf' C-m
+      tmux send-keys -t remote 'export OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/protobuf' C-m
       tmux send-keys -t remote 'export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://xray.${var.aws_region}.amazonaws.com/v1/traces' C-m
+      tmux send-keys -t remote 'export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://logs.${var.aws_region}.amazonaws.com/v1/logs' C-m
+      tmux send-keys -t remote 'export OTEL_EXPORTER_OTLP_LOGS_HEADERS=x-aws-log-group=${var.application_logs_log_group},x-aws-log-stream=default' C-m
       tmux send-keys -t remote 'export OTEL_NODE_DISABLED_INSTRUMENTATIONS=fs,dns,express' C-m
       tmux send-keys -t remote 'export OTEL_SERVICE_NAME=node-sample-remote-application-${var.test_id}' C-m
       tmux send-keys -t remote 'export OTEL_TRACES_SAMPLER=always_on' C-m

terraform/node/ec2/adot-sigv4/variables.tf

@@ -45,4 +45,8 @@ variable "get_adot_instrumentation_command" {
 
 variable "canary_type" {
   default = "node-ec2-default"
+}
+
+variable "application_logs_log_group" {
+  default = "otlp_logs"
 }

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -454,23 +454,26 @@ public enum PredefinedExpectedTemplate implements FileConfig {
 
   /** Node EC2 ADOT SigV4 (Stand Alone ADOT) Test Case Validations */
   NODE_EC2_ADOT_SIGV4_OUTGOING_HTTP_CALL_LOG(
-      "/expected-data-template/node/ec2/adot-sigv4/outgoing-http-call-log.mustache"),
+      "/expected-data-template/node/ec2/adot-aws-otlp/outgoing-http-call-log.mustache"),
   NODE_EC2_ADOT_SIGV4_OUTGOING_HTTP_CALL_METRIC(
-      "/expected-data-template/node/ec2/adot-sigv4/outgoing-http-call-metric.mustache"),
+      "/expected-data-template/node/ec2/adot-aws-otlp/outgoing-http-call-metric.mustache"),
   NODE_EC2_ADOT_SIGV4_OUTGOING_HTTP_CALL_TRACE(
-      "/expected-data-template/node/ec2/adot-sigv4/outgoing-http-call-trace.mustache"),
+      "/expected-data-template/node/ec2/adot-aws-otlp/outgoing-http-call-trace.mustache"),
 
-  NODE_EC2_ADOT_SIGV4_AWS_SDK_CALL_LOG("/expected-data-template/node/ec2/adot-sigv4/aws-sdk-call-log.mustache"),
-  NODE_EC2_ADOT_SIGV4_AWS_SDK_CALL_METRIC("/expected-data-template/node/ec2/adot-sigv4/aws-sdk-call-metric.mustache"),
-  NODE_EC2_ADOT_SIGV4_AWS_SDK_CALL_TRACE("/expected-data-template/node/ec2/adot-sigv4/aws-sdk-call-trace.mustache"),
+  NODE_EC2_ADOT_SIGV4_AWS_SDK_CALL_LOG("/expected-data-template/node/ec2/adot-aws-otlp/aws-sdk-call-log.mustache"),
+  NODE_EC2_ADOT_SIGV4_AWS_SDK_CALL_METRIC("/expected-data-template/node/ec2/adot-aws-otlp/aws-sdk-call-metric.mustache"),
+  NODE_EC2_ADOT_SIGV4_AWS_SDK_CALL_TRACE("/expected-data-template/node/ec2/adot-aws-otlp/aws-sdk-call-trace.mustache"),
 
-  NODE_EC2_ADOT_SIGV4_REMOTE_SERVICE_LOG("/expected-data-template/node/ec2/adot-sigv4/remote-service-log.mustache"),
-  NODE_EC2_ADOT_SIGV4_REMOTE_SERVICE_METRIC("/expected-data-template/node/ec2/adot-sigv4/remote-service-metric.mustache"),
-  NODE_EC2_ADOT_SIGV4_REMOTE_SERVICE_TRACE("/expected-data-template/node/ec2/adot-sigv4/remote-service-trace.mustache"),
+  NODE_EC2_ADOT_SIGV4_REMOTE_SERVICE_LOG("/expected-data-template/node/ec2/adot-aws-otlp/remote-service-log.mustache"),
+  NODE_EC2_ADOT_SIGV4_REMOTE_SERVICE_METRIC("/expected-data-template/node/ec2/adot-aws-otlp/remote-service-metric.mustache"),
+  NODE_EC2_ADOT_SIGV4_REMOTE_SERVICE_TRACE("/expected-data-template/node/ec2/adot-aws-otlp/remote-service-trace.mustache"),
 
-  NODE_EC2_ADOT_SIGV4_CLIENT_CALL_LOG("/expected-data-template/node/ec2/adot-sigv4/client-call-log.mustache"),
-  NODE_EC2_ADOT_SIGV4_CLIENT_CALL_METRIC("/expected-data-template/node/ec2/adot-sigv4/client-call-metric.mustache"),
-  NODE_EC2_ADOT_SIGV4_CLIENT_CALL_TRACE("/expected-data-template/node/ec2/adot-sigv4/client-call-trace.mustache"),
+  NODE_EC2_ADOT_SIGV4_CLIENT_CALL_LOG("/expected-data-template/node/ec2/adot-aws-otlp/client-call-log.mustache"),
+  NODE_EC2_ADOT_SIGV4_CLIENT_CALL_METRIC("/expected-data-template/node/ec2/adot-aws-otlp/client-call-metric.mustache"),
+  NODE_EC2_ADOT_SIGV4_CLIENT_CALL_TRACE("/expected-data-template/node/ec2/adot-aws-otlp/client-call-trace.mustache"),
+
+  /** Node EC2 ADOT SigV4 Log Exporter Test Case Validation */
+  NODE_EC2_ADOT_OTLP_LOG("/expected-data-template/node/ec2/adot-aws-otlp/application-log.mustache"),
 
   /** Node ECS Test Case Validations */
   NODE_ECS_HC_CALL_LOG("/expected-data-template/node/ecs/hc-log.mustache"),

validator/src/main/resources/expected-data-template/node/ec2/adot-aws-otlp/application-log.mustache

@@ -0,0 +1,16 @@
+[{
+  "resource": {
+    "attributes": {
+      "service.name": "{{serviceName}}",
+      "cloud.provider": "aws",
+      "cloud.region": "{{region}}",
+      "cloud.account.id": "{{accountId}}",
+      "cloud.platform": "aws_ec2"
+    }
+  },
+  "severityNumber": "^[0-9]+$",
+  "severityText": "{{severityText}}",
+  "body": "This is a custom log for validation testing",
+  "traceId": "{{traceId}}",
+  "spanId": "{{spanId}}"
+}]

validator/src/main/resources/validations/node/ec2/adot-aws-otlp/application-log-validation.yml

@@ -0,0 +1,3 @@
+-
+  validationType: "cw-log"
+  expectedLogStructureTemplate: "NODE_EC2_ADOT_OTLP_LOG"