[Python] Custom Metrics E2E Test

.github/workflows/java-ec2-adot-sigv4-test.yml

@@ -34,7 +34,7 @@ permissions:
   contents: read
 
 env:
-  E2E_TEST_AWS_REGION: 'us-west-2' # Test uses us-west-2 in the us-east-1 accoun
+  E2E_TEST_AWS_REGION: 'us-west-2' # Test uses us-west-2 in the us-east-1 account
   CALLER_WORKFLOW_NAME: ${{ inputs.caller-workflow-name }}
   METRIC_NAMESPACE: ApplicationSignals
   JAVA_VERSION: ${{ inputs.java-version }}

.github/workflows/python-ec2-default-test.yml

@@ -32,6 +32,10 @@ on:
         required: false
         type: string
         default: 'github'
+      custom-metrics-enabled:
+        required: false
+        type: boolean
+        default: false
     outputs:
       job-started:
         value: ${{ jobs.python-ec2-default.outputs.job-started }}
@@ -49,6 +53,7 @@ env:
   CPU_ARCHITECTURE: ${{ inputs.cpu-architecture }}
   ADOT_WHEEL_NAME: ${{ inputs.staging-wheel-name }}
   OTEL_SOURCE: ${{ inputs.otel-source }}
+  CUSTOM_METRICS_ENABLED: ${{ inputs.custom-metrics-enabled }}
   SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/python-sample-app.zip
   E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
@@ -171,8 +176,8 @@ jobs:
               -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \
               -var="language_version=${{ env.PYTHON_VERSION }}" \
               -var="cpu_architecture=${{ env.CPU_ARCHITECTURE }}" \
-            || deployment_failed=$?
-
+              -var="custom_metrics_enabled=${{ env.CUSTOM_METRICS_ENABLED }}" \
+              || deployment_failed=1
             if [ $deployment_failed -eq 1 ]; then
               echo "Terraform deployment was unsuccessful. Will attempt to retry deployment."
             fi
@@ -252,9 +257,26 @@ jobs:
           --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
           --rollup'
 
+      - name: Validate CWAgent metrics
+        id: cwagent-metric-validation
+        if: (success() || (env.CUSTOM_METRICS_ENABLED == 'true' && steps.log-validation.outcome == 'failure')) && !cancelled()
+        run: ./gradlew validator:run --args='-c python/ec2/custom-metrics/custom-metric-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
+          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
+          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
+          --region ${{ env.E2E_TEST_AWS_REGION }}
+          --metric-namespace CWAgent
+          --log-group ${{ env.LOG_GROUP_NAME }}
+          --service-name python-sample-application-${{ env.TESTING_ID }}
+          --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
+          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
+          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
+          --rollup'
+
       - name: Validate generated traces
         id: trace-validation
-        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' || (env.CUSTOM_METRICS_ENABLED == 'true' && steps.cwagent-metric-validation.outcome == 'failure')) && !cancelled()
         run: ./gradlew validator:run --args='-c python/ec2/default/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
@@ -281,7 +303,7 @@ jobs:
         if: always()
         id: validation-result
         run: |
-          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
+          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ env.CUSTOM_METRICS_ENABLED }}" = "true" ] && [ "${{ steps.cwagent-metric-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
             echo "validation-result=success" >> $GITHUB_OUTPUT
           else
             echo "validation-result=failure" >> $GITHUB_OUTPUT

sample-apps/python/django_frontend_service/frontend_service_app/views.py

@@ -5,17 +5,22 @@
 import base64
 import threading
 import time
-
+import random
 import boto3
 import pymysql
 import requests
 import schedule
 from django.http import HttpResponse, JsonResponse
-from opentelemetry import trace
+from opentelemetry import trace, metrics
 from opentelemetry.trace.span import format_trace_id
 
 logger = logging.getLogger(__name__)
 
+#python equivalent of Meter meter = GlobalOpenTelemetry.getMeter("myMeter"); for custom metrics
+meter = metrics.get_meter("myMeter")
+custom_export_counter = meter.create_counter("custom_export_counter", unit="1", description="Custom export counter")
+test_histogram = meter.create_histogram("test_histogram", description="Test histogram")
+
 should_send_local_root_client_call = False
 lock = threading.Lock()
 def run_local_root_client_call_recurring_service():
@@ -50,6 +55,11 @@ def healthcheck(request):
     return HttpResponse("healthcheck")
 
 def aws_sdk_call(request):
+
+    # Increment counter/histogram
+    custom_export_counter.add(1, {"operation.type": "custom_export_1"})
+    test_histogram.record(random.randint(100, 1000), {"operation.type": "histogram"})
+
     bucket_name = "e2e-test-bucket-name"
 
     # Add a unique test ID to bucketname to associate buckets to specific test runs

terraform/python/ec2/adot-genai/variables.tf

@@ -14,7 +14,7 @@
 # -------------------------------------------------------------------------
 
 variable "aws_region" {
-  default = "us-west-2"
+  default = "us-east-1"
 }
 
 variable "test_id" {

terraform/python/ec2/default/amazon-cloudwatch-agent.json

@@ -10,7 +10,11 @@
   },
   "logs": {
     "metrics_collected": {
-      "application_signals": {}
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
     }
   }
 }

terraform/python/ec2/default/main.tf

@@ -170,15 +170,20 @@ resource "null_resource" "main_service_setup" {
       export DJANGO_SETTINGS_MODULE="django_frontend_service.settings"
       export OTEL_PYTHON_DISTRO="aws_distro"
       export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
-      export OTEL_METRICS_EXPORTER=none
+      export OTEL_METRICS_EXPORTER=otlp
       export OTEL_TRACES_EXPORTER=otlp
       export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true
       export OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4315
       export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4315
       export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
       export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc
+      export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=localhost:4317
+      export OTEL_EXPORTER_OTLP_INSECURE=true
       export OTEL_SERVICE_NAME=python-sample-application-${var.test_id}
       export OTEL_TRACES_SAMPLER=always_on
+      export OTEL_RESOURCE_ATTRIBUTES="Service=python-sample-application-${var.test_id},Environment=ec2:default"
+      export AWS_REGION='${var.aws_region}'
+      export CUSTOM_METRICS_ENABLED='${var.custom_metrics_enabled}'
       python${var.language_version} manage.py migrate
       nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8000 --noreload &
 
@@ -276,7 +281,7 @@ resource "null_resource" "remote_service_setup" {
 
       # Copy in CW Agent configuration
       agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
-      echo $agent_config > amazon-cloudwatch-agent.json
+      echo "$agent_config" > amazon-cloudwatch-agent.json
 
       # Get and run CW agent rpm
       ${var.get_cw_agent_rpm_command}

terraform/python/ec2/default/variables.tf

@@ -47,4 +47,12 @@ variable "language_version" {
 
 variable "cpu_architecture" {
   default = "x86_64"
+}
+
+#Adding Custom Metrics Variables
+
+variable "custom_metrics_enabled" {
+  description = "Enable custom OTEL metrics in the sample application"
+  type        = bool
+  default     = false
 }

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -210,6 +210,9 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   PYTHON_EC2_DEFAULT_CLIENT_CALL_METRIC("/expected-data-template/python/ec2/default/client-call-metric.mustache"),
   PYTHON_EC2_DEFAULT_CLIENT_CALL_TRACE("/expected-data-template/python/ec2/default/client-call-trace.mustache"),
 
+  /** Python EC2 Default Custom Metrics Test Case Validations */
+  PYTHON_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC("/expected-data-template/python/ec2/custom-metrics/aws-otel-custom-metrics.mustache"),
+
   /** Python EC2 Asg Test Case Validations */
   PYTHON_EC2_ASG_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/ec2/asg/outgoing-http-call-log.mustache"),
   PYTHON_EC2_ASG_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/ec2/asg/outgoing-http-call-metric.mustache"),

validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java

@@ -99,8 +99,8 @@ public void validate() throws Exception {
             validateAnyMetricExists();
             return;
           }
-          // We will query the Service, RemoteService, and RemoteTarget dimensions to ensure we
-          // get all metrics from all aggregations, specifically the [RemoteService] aggregation.
+          // Query the Service, RemoteService, and RemoteTarget dimensions to ensure we
+          // Get all metrics from all aggregations, specifically the [RemoteService] aggregation.
           List<String> serviceNames =
               Lists.newArrayList(
                   context.getServiceName(), context.getRemoteServiceDeploymentName());
@@ -158,9 +158,9 @@ private void addMetrics(
       List<Metric> actualMetricList)
       throws Exception {
     for (String dimensionValue : dimensionValues) {
-      actualMetricList.addAll(
-          this.listMetricFromCloudWatch(
-              cloudWatchService, expectedMetricList, dimensionName, dimensionValue));
+      List<Metric> foundMetrics = this.listMetricFromCloudWatch(
+              cloudWatchService, expectedMetricList, dimensionName, dimensionValue);
+      actualMetricList.addAll(foundMetrics);
     }
   }
 
@@ -245,9 +245,10 @@ private List<Metric> listMetricFromCloudWatch(
     // search by metric name
     List<Metric> result = new ArrayList<>();
     for (String metricName : metricNameMap.keySet()) {
-      result.addAll(
-          cloudWatchService.listMetrics(
-              metricNameMap.get(metricName), metricName, dimensionKey, dimensionValue));
+      String namespace = metricNameMap.get(metricName);
+
+      List<Metric> metrics = cloudWatchService.listMetrics(namespace, metricName, dimensionKey, dimensionValue);
+      result.addAll(metrics);
     }
     return result;
   }

validator/src/main/resources/expected-data-template/python/ec2/custom-metrics/aws-otel-custom-metrics.mustache

@@ -0,0 +1,112 @@
+# OpenTelemetry Custom Metrics Validation Templates - AWS SDK Call Only
+# ANY_VALUE defines a string to = 'ANY_VALUE' to pass validation testing
+-
+  metricName: custom_export_counter
+  namespace: CWAgent
+  dimensions:
+    -
+      name: Environment
+      value: ec2:default
+    -
+      name: Service
+      value: {{serviceName}}
+    -
+      name: aws.local.service
+      value: {{serviceName}}
+    -
+      name: cloud.account.id
+      value: 654654176582
+    -
+      name: cloud.availability_zone
+      value: ANY_VALUE
+    -
+      name: cloud.platform
+      value: aws_ec2
+    -
+      name: cloud.provider
+      value: aws
+    -
+      name: cloud.region
+      value: us-east-1
+    -
+      name: host.id
+      value: ANY_VALUE
+    -
+      name: host.name
+      value: ANY_VALUE
+    -
+      name: host.type
+      value: t3.micro
+    -
+      name: operation.type
+      value: custom_export_1
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: telemetry.auto.version
+      value: ANY_VALUE
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE
+-
+  metricName: test_histogram
+  namespace: CWAgent
+  dimensions:
+    -
+      name: Environment
+      value: ec2:default
+    -
+      name: Service
+      value: {{serviceName}}
+    -
+      name: aws.local.service
+      value: {{serviceName}}
+    -
+      name: cloud.account.id
+      value: 654654176582
+    -
+      name: cloud.availability_zone
+      value: ANY_VALUE
+    -
+      name: cloud.platform
+      value: aws_ec2
+    -
+      name: cloud.provider
+      value: aws
+    -
+      name: cloud.region
+      value: us-east-1
+    -
+      name: host.id
+      value: ANY_VALUE
+    -
+      name: host.name
+      value: ANY_VALUE
+    -
+      name: host.type
+      value: t3.micro
+    -
+      name: operation.type
+      value: histogram
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: telemetry.auto.version
+      value: ANY_VALUE
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE

validator/src/main/resources/validations/python/ec2/custom-metrics/custom-metric-validation.yml

@@ -0,0 +1,6 @@
+-
+  validationType: "cw-metric"
+  httpPath: "aws-sdk-call"
+  httpMethod: "get"
+  callingType: "http-with-query"
+  expectedMetricTemplate: "PYTHON_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC"