diff --git a/.github/workflows/java-ec2-adot-sigv4-test.yml b/.github/workflows/java-ec2-adot-sigv4-test.yml index 8a7973dcd..ab42a5a38 100644 --- a/.github/workflows/java-ec2-adot-sigv4-test.yml +++ b/.github/workflows/java-ec2-adot-sigv4-test.yml @@ -34,7 +34,7 @@ permissions: contents: read env: - E2E_TEST_AWS_REGION: 'us-west-2' # Test uses us-west-2 in the us-east-1 accoun + E2E_TEST_AWS_REGION: 'us-west-2' # Test uses us-west-2 in the us-east-1 account CALLER_WORKFLOW_NAME: ${{ inputs.caller-workflow-name }} METRIC_NAMESPACE: ApplicationSignals JAVA_VERSION: ${{ inputs.java-version }} diff --git a/.github/workflows/python-ec2-default-test.yml b/.github/workflows/python-ec2-default-test.yml index 4fb5aa575..948ced649 100644 --- a/.github/workflows/python-ec2-default-test.yml +++ b/.github/workflows/python-ec2-default-test.yml @@ -171,8 +171,7 @@ jobs: -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \ -var="language_version=${{ env.PYTHON_VERSION }}" \ -var="cpu_architecture=${{ env.CPU_ARCHITECTURE }}" \ - || deployment_failed=$? - + || deployment_failed=1 if [ $deployment_failed -eq 1 ]; then echo "Terraform deployment was unsuccessful. Will attempt to retry deployment." fi @@ -252,9 +251,26 @@ jobs: --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} --rollup' + - name: Validate CWAgent metrics + id: cwagent-metric-validation + if: (success() || steps.log-validation.outcome == 'failure') && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/default/custom-metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} + --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001 + --region ${{ env.E2E_TEST_AWS_REGION }} + --metric-namespace CWAgent + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name python-sample-application-${{ env.TESTING_ID }} + --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }} + --query-string ip=${{ env.REMOTE_SERVICE_IP }} + --instance-ami ${{ env.EC2_INSTANCE_AMI }} + --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }} + --rollup' + - name: Validate generated traces id: trace-validation - if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled() + if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' || steps.cwagent-metric-validation.outcome == 'failure') && !cancelled() run: ./gradlew validator:run --args='-c python/ec2/default/trace-validation.yml --testing-id ${{ env.TESTING_ID }} --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }} @@ -281,7 +297,7 @@ jobs: if: always() id: validation-result run: | - if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then + if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.cwagent-metric-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then echo "validation-result=success" >> $GITHUB_OUTPUT else echo "validation-result=failure" >> $GITHUB_OUTPUT diff --git a/sample-apps/python/django_frontend_service/frontend_service_app/views.py b/sample-apps/python/django_frontend_service/frontend_service_app/views.py index 57c264f3c..bd49bdb99 100644 --- a/sample-apps/python/django_frontend_service/frontend_service_app/views.py +++ b/sample-apps/python/django_frontend_service/frontend_service_app/views.py @@ -5,17 +5,22 @@ import base64 import threading import time - +import random import boto3 import pymysql import requests import schedule from django.http import HttpResponse, JsonResponse -from opentelemetry import trace +from opentelemetry import trace, metrics from opentelemetry.trace.span import format_trace_id logger = logging.getLogger(__name__) +#python equivalent of Meter meter = GlobalOpenTelemetry.getMeter("myMeter"); for custom metrics +meter = metrics.get_meter("myMeter") +custom_export_counter = meter.create_counter("custom_export_counter", unit="1", description="Custom export counter") +test_histogram = meter.create_histogram("test_histogram", description="Test histogram") + should_send_local_root_client_call = False lock = threading.Lock() def run_local_root_client_call_recurring_service(): @@ -50,6 +55,11 @@ def healthcheck(request): return HttpResponse("healthcheck") def aws_sdk_call(request): + + # Increment counter/histogram + custom_export_counter.add(1, {"operation.type": "custom_export_1"}) + test_histogram.record(random.randint(100, 1000), {"operation.type": "histogram"}) + bucket_name = "e2e-test-bucket-name" # Add a unique test ID to bucketname to associate buckets to specific test runs diff --git a/terraform/python/ec2/default/amazon-cloudwatch-agent.json b/terraform/python/ec2/default/amazon-cloudwatch-agent.json index a98a40d36..f65bfe32e 100644 --- a/terraform/python/ec2/default/amazon-cloudwatch-agent.json +++ b/terraform/python/ec2/default/amazon-cloudwatch-agent.json @@ -10,7 +10,11 @@ }, "logs": { "metrics_collected": { - "application_signals": {} + "application_signals": {}, + "otlp": { + "grpc_endpoint": "0.0.0.0:4317", + "http_endpoint": "0.0.0.0:4318" + } } } } \ No newline at end of file diff --git a/terraform/python/ec2/default/main.tf b/terraform/python/ec2/default/main.tf index f5f254232..142545dde 100644 --- a/terraform/python/ec2/default/main.tf +++ b/terraform/python/ec2/default/main.tf @@ -170,15 +170,19 @@ resource "null_resource" "main_service_setup" { export DJANGO_SETTINGS_MODULE="django_frontend_service.settings" export OTEL_PYTHON_DISTRO="aws_distro" export OTEL_PYTHON_CONFIGURATOR="aws_configurator" - export OTEL_METRICS_EXPORTER=none + export OTEL_METRICS_EXPORTER=otlp export OTEL_TRACES_EXPORTER=otlp export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true export OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4315 export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4315 export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc + export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=localhost:4317 + export OTEL_EXPORTER_OTLP_INSECURE=true export OTEL_SERVICE_NAME=python-sample-application-${var.test_id} export OTEL_TRACES_SAMPLER=always_on + export OTEL_RESOURCE_ATTRIBUTES="Service=python-sample-application-${var.test_id},Environment=ec2:default" + export AWS_REGION='${var.aws_region}' python${var.language_version} manage.py migrate nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8000 --noreload & @@ -276,7 +280,7 @@ resource "null_resource" "remote_service_setup" { # Copy in CW Agent configuration agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}' - echo $agent_config > amazon-cloudwatch-agent.json + echo "$agent_config" > amazon-cloudwatch-agent.json # Get and run CW agent rpm ${var.get_cw_agent_rpm_command} diff --git a/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java b/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java index 3523f2131..9346119ee 100644 --- a/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java +++ b/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java @@ -210,6 +210,9 @@ public enum PredefinedExpectedTemplate implements FileConfig { PYTHON_EC2_DEFAULT_CLIENT_CALL_METRIC("/expected-data-template/python/ec2/default/client-call-metric.mustache"), PYTHON_EC2_DEFAULT_CLIENT_CALL_TRACE("/expected-data-template/python/ec2/default/client-call-trace.mustache"), + /** Python EC2 Default Custom Metrics Test Case Validations */ + PYTHON_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC("/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache"), + /** Python EC2 Asg Test Case Validations */ PYTHON_EC2_ASG_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/ec2/asg/outgoing-http-call-log.mustache"), PYTHON_EC2_ASG_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/ec2/asg/outgoing-http-call-metric.mustache"), diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java index dca2d964f..67ebf8eb6 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java @@ -99,8 +99,8 @@ public void validate() throws Exception { validateAnyMetricExists(); return; } - // We will query the Service, RemoteService, and RemoteTarget dimensions to ensure we - // get all metrics from all aggregations, specifically the [RemoteService] aggregation. + // Query the Service, RemoteService, and RemoteTarget dimensions to ensure we + // Get all metrics from all aggregations, specifically the [RemoteService] aggregation. List serviceNames = Lists.newArrayList( context.getServiceName(), context.getRemoteServiceDeploymentName()); @@ -158,9 +158,9 @@ private void addMetrics( List actualMetricList) throws Exception { for (String dimensionValue : dimensionValues) { - actualMetricList.addAll( - this.listMetricFromCloudWatch( - cloudWatchService, expectedMetricList, dimensionName, dimensionValue)); + List foundMetrics = this.listMetricFromCloudWatch( + cloudWatchService, expectedMetricList, dimensionName, dimensionValue); + actualMetricList.addAll(foundMetrics); } } @@ -245,9 +245,10 @@ private List listMetricFromCloudWatch( // search by metric name List result = new ArrayList<>(); for (String metricName : metricNameMap.keySet()) { - result.addAll( - cloudWatchService.listMetrics( - metricNameMap.get(metricName), metricName, dimensionKey, dimensionValue)); + String namespace = metricNameMap.get(metricName); + + List metrics = cloudWatchService.listMetrics(namespace, metricName, dimensionKey, dimensionValue); + result.addAll(metrics); } return result; } diff --git a/validator/src/main/resources/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache b/validator/src/main/resources/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache new file mode 100644 index 000000000..f78f91307 --- /dev/null +++ b/validator/src/main/resources/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache @@ -0,0 +1,112 @@ +# OpenTelemetry Custom Metrics Validation Templates - AWS SDK Call Only +# ANY_VALUE defines a string to = 'ANY_VALUE' to pass validation testing +- + metricName: custom_export_counter + namespace: CWAgent + dimensions: + - + name: Environment + value: ec2:default + - + name: Service + value: {{serviceName}} + - + name: aws.local.service + value: {{serviceName}} + - + name: cloud.account.id + value: ANY_VALUE + - + name: cloud.availability_zone + value: ANY_VALUE + - + name: cloud.platform + value: aws_ec2 + - + name: cloud.provider + value: aws + - + name: cloud.region + value: ANY_VALUE + - + name: host.id + value: ANY_VALUE + - + name: host.name + value: ANY_VALUE + - + name: host.type + value: ANY_VALUE + - + name: operation.type + value: custom_export_1 + - + name: service.name + value: {{serviceName}} + - + name: telemetry.auto.version + value: ANY_VALUE + - + name: telemetry.sdk.language + value: python + - + name: telemetry.sdk.name + value: opentelemetry + - + name: telemetry.sdk.version + value: ANY_VALUE +- + metricName: test_histogram + namespace: CWAgent + dimensions: + - + name: Environment + value: ec2:default + - + name: Service + value: {{serviceName}} + - + name: aws.local.service + value: {{serviceName}} + - + name: cloud.account.id + value: ANY_VALUE + - + name: cloud.availability_zone + value: ANY_VALUE + - + name: cloud.platform + value: aws_ec2 + - + name: cloud.provider + value: aws + - + name: cloud.region + value: ANY_VALUE + - + name: host.id + value: ANY_VALUE + - + name: host.name + value: ANY_VALUE + - + name: host.type + value: ANY_VALUE + - + name: operation.type + value: histogram + - + name: service.name + value: {{serviceName}} + - + name: telemetry.auto.version + value: ANY_VALUE + - + name: telemetry.sdk.language + value: python + - + name: telemetry.sdk.name + value: opentelemetry + - + name: telemetry.sdk.version + value: ANY_VALUE \ No newline at end of file diff --git a/validator/src/main/resources/validations/python/ec2/default/custom-metric-validation.yml b/validator/src/main/resources/validations/python/ec2/default/custom-metric-validation.yml new file mode 100644 index 000000000..4513a9405 --- /dev/null +++ b/validator/src/main/resources/validations/python/ec2/default/custom-metric-validation.yml @@ -0,0 +1,6 @@ +- + validationType: "cw-metric" + httpPath: "aws-sdk-call" + httpMethod: "get" + callingType: "http-with-query" + expectedMetricTemplate: "PYTHON_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC"