From ce8f2bf7a391b1e60c27578144869c71e535778f Mon Sep 17 00:00:00 2001 From: Mingjie Shao Date: Thu, 12 Sep 2024 17:15:51 +0100 Subject: [PATCH 1/6] Revert "Enhance Logs E2E test and bump up MySQL library to 8.4 (#221)" This reverts commit 3502e733750aa4105c6ebc5167f8aeb410a3cafd. --- sample-apps/springboot/build.gradle.kts | 2 +- .../com/amazon/aoc/validators/CWLogValidator.java | 10 ++-------- .../java/eks/rds-mysql-trace.mustache | 14 ++++---------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/sample-apps/springboot/build.gradle.kts b/sample-apps/springboot/build.gradle.kts index c14ede174..242e72135 100644 --- a/sample-apps/springboot/build.gradle.kts +++ b/sample-apps/springboot/build.gradle.kts @@ -37,7 +37,7 @@ dependencies { implementation("io.opentelemetry:opentelemetry-api:1.34.1") implementation("software.amazon.awssdk:s3") implementation("software.amazon.awssdk:sts") - implementation("com.mysql:mysql-connector-j:8.4.0") + implementation("com.mysql:mysql-connector-j:8.0.33") } jib { diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java index 3c4bbceec..401281cb0 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java @@ -66,11 +66,9 @@ public void validate() throws Exception { String operation = (String) expectedAttributes.get("Operation"); String remoteService = (String) expectedAttributes.get("RemoteService"); String remoteOperation = (String) expectedAttributes.get("RemoteOperation"); - String remoteResourceType = (String) expectedAttributes.get("RemoteResourceType"); - String remoteResourceIdentifier = (String) expectedAttributes.get("RemoteResourceIdentifier"); Map actualLog = - this.getActualLog(operation, remoteService, remoteOperation, remoteResourceType, remoteResourceIdentifier); + this.getActualLog(operation, remoteService, remoteOperation); log.info("Value of an actual log: {}", actualLog); if (actualLog == null) throw new BaseException(ExceptionCode.EXPECTED_LOG_NOT_FOUND); @@ -128,7 +126,7 @@ private JsonifyArrayList> getExpectedAttributes() throws Exc } private Map getActualLog( - String operation, String remoteService, String remoteOperation, String remoteResourceType, String remoteResourceIdentifier) throws Exception { + String operation, String remoteService, String remoteOperation) throws Exception { String dependencyFilter = null; // Dependency calls will have the remoteService and remoteOperation attribute, but service calls @@ -141,10 +139,6 @@ private Map getActualLog( dependencyFilter = String.format("&& ($.RemoteService = \"%s\") && ($.RemoteOperation = \"%s\")", remoteService, remoteOperation); } - if (remoteResourceType != null && remoteResourceIdentifier != null) { - dependencyFilter += String.format(" && ($.RemoteResourceType = %%%s%%) && ($.RemoteResourceIdentifier = %%%s%%)", remoteResourceType, remoteResourceIdentifier); - } - String filterPattern = String.format("{ ($.Service = %s) && ($.Operation = \"%s\") %s }", context.getServiceName(), operation, dependencyFilter); log.info("Filter Pattern for Log Search: " + filterPattern); diff --git a/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache b/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache index d30d80aa5..7843a9d9e 100644 --- a/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache +++ b/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache @@ -26,8 +26,8 @@ "default": { "EC2.AutoScalingGroup": "^eks-.+", "EKS.Cluster": "^{{platformInfo}}$", - "K8s.Namespace": "^{{appNamespace}}$", - "otel.resource.K8s.Workload": "^sample-app-deployment-{{testingId}}$", + "K8s.Namespace": "^{{appNamespace}}", + "otel.resource.K8s.Workload": "^sample-app-deployment-{{testingId}}", "otel.resource.K8s.Node": "^i-[A-Za-z0-9]{17}$", "otel.resource.K8s.Pod": "^sample-app-deployment-{{testingId}}(-[A-Za-z0-9]*)*$", "otel.resource.host.name": "^ip(-[0-9]{1,3}){4}.*$", @@ -50,12 +50,6 @@ "aws.local.operation": "^GET /mysql$" }, "subsegments": [ - { - "name": "^mysql$", - "sql": { - "url": "^Create connection$" - } - }, { "name": "^mysql$", "sql": { @@ -67,7 +61,7 @@ "aws.remote.operation": "^SELECT$", "aws.local.operation": "^GET /mysql$", "aws.remote.resource.type": "^DB::Connection$", - "aws.remote.resource.identifier": "^{{remoteResourceIdentifier}}$", + "aws.remote.resource.identifier": "{{remoteResourceIdentifier}}", "aws.remote.service": "^mysql$", "aws.local.service": "^{{serviceName}}$", "aws.local.environment": "^eks:{{platformInfo}}/{{appNamespace}}$" @@ -89,4 +83,4 @@ } ] } -] +] \ No newline at end of file From 45413d5822effcdb725810027059455ba0173605 Mon Sep 17 00:00:00 2001 From: ektabj Date: Wed, 11 Sep 2024 22:49:30 +0100 Subject: [PATCH 2/6] Enhance Logs E2E test and bump up MySQL library to 8.4 (#221) --- sample-apps/springboot/build.gradle.kts | 2 +- .../com/amazon/aoc/validators/CWLogValidator.java | 10 ++++++++-- .../java/eks/rds-mysql-trace.mustache | 14 ++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/sample-apps/springboot/build.gradle.kts b/sample-apps/springboot/build.gradle.kts index 242e72135..c14ede174 100644 --- a/sample-apps/springboot/build.gradle.kts +++ b/sample-apps/springboot/build.gradle.kts @@ -37,7 +37,7 @@ dependencies { implementation("io.opentelemetry:opentelemetry-api:1.34.1") implementation("software.amazon.awssdk:s3") implementation("software.amazon.awssdk:sts") - implementation("com.mysql:mysql-connector-j:8.0.33") + implementation("com.mysql:mysql-connector-j:8.4.0") } jib { diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java index 401281cb0..3c4bbceec 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java @@ -66,9 +66,11 @@ public void validate() throws Exception { String operation = (String) expectedAttributes.get("Operation"); String remoteService = (String) expectedAttributes.get("RemoteService"); String remoteOperation = (String) expectedAttributes.get("RemoteOperation"); + String remoteResourceType = (String) expectedAttributes.get("RemoteResourceType"); + String remoteResourceIdentifier = (String) expectedAttributes.get("RemoteResourceIdentifier"); Map actualLog = - this.getActualLog(operation, remoteService, remoteOperation); + this.getActualLog(operation, remoteService, remoteOperation, remoteResourceType, remoteResourceIdentifier); log.info("Value of an actual log: {}", actualLog); if (actualLog == null) throw new BaseException(ExceptionCode.EXPECTED_LOG_NOT_FOUND); @@ -126,7 +128,7 @@ private JsonifyArrayList> getExpectedAttributes() throws Exc } private Map getActualLog( - String operation, String remoteService, String remoteOperation) throws Exception { + String operation, String remoteService, String remoteOperation, String remoteResourceType, String remoteResourceIdentifier) throws Exception { String dependencyFilter = null; // Dependency calls will have the remoteService and remoteOperation attribute, but service calls @@ -139,6 +141,10 @@ private Map getActualLog( dependencyFilter = String.format("&& ($.RemoteService = \"%s\") && ($.RemoteOperation = \"%s\")", remoteService, remoteOperation); } + if (remoteResourceType != null && remoteResourceIdentifier != null) { + dependencyFilter += String.format(" && ($.RemoteResourceType = %%%s%%) && ($.RemoteResourceIdentifier = %%%s%%)", remoteResourceType, remoteResourceIdentifier); + } + String filterPattern = String.format("{ ($.Service = %s) && ($.Operation = \"%s\") %s }", context.getServiceName(), operation, dependencyFilter); log.info("Filter Pattern for Log Search: " + filterPattern); diff --git a/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache b/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache index 7843a9d9e..d30d80aa5 100644 --- a/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache +++ b/validator/src/main/resources/expected-data-template/java/eks/rds-mysql-trace.mustache @@ -26,8 +26,8 @@ "default": { "EC2.AutoScalingGroup": "^eks-.+", "EKS.Cluster": "^{{platformInfo}}$", - "K8s.Namespace": "^{{appNamespace}}", - "otel.resource.K8s.Workload": "^sample-app-deployment-{{testingId}}", + "K8s.Namespace": "^{{appNamespace}}$", + "otel.resource.K8s.Workload": "^sample-app-deployment-{{testingId}}$", "otel.resource.K8s.Node": "^i-[A-Za-z0-9]{17}$", "otel.resource.K8s.Pod": "^sample-app-deployment-{{testingId}}(-[A-Za-z0-9]*)*$", "otel.resource.host.name": "^ip(-[0-9]{1,3}){4}.*$", @@ -50,6 +50,12 @@ "aws.local.operation": "^GET /mysql$" }, "subsegments": [ + { + "name": "^mysql$", + "sql": { + "url": "^Create connection$" + } + }, { "name": "^mysql$", "sql": { @@ -61,7 +67,7 @@ "aws.remote.operation": "^SELECT$", "aws.local.operation": "^GET /mysql$", "aws.remote.resource.type": "^DB::Connection$", - "aws.remote.resource.identifier": "{{remoteResourceIdentifier}}", + "aws.remote.resource.identifier": "^{{remoteResourceIdentifier}}$", "aws.remote.service": "^mysql$", "aws.local.service": "^{{serviceName}}$", "aws.local.environment": "^eks:{{platformInfo}}/{{appNamespace}}$" @@ -83,4 +89,4 @@ } ] } -] \ No newline at end of file +] From e0e3419cabe112cb905660c789cd2c4c80b2b826 Mon Sep 17 00:00:00 2001 From: Mingjie Shao Date: Tue, 24 Sep 2024 11:35:58 +0100 Subject: [PATCH 3/6] Allow workflow run in forked repo --- .github/workflows/python-eks-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-eks-test.yml b/.github/workflows/python-eks-test.yml index cb27293d7..eaba6653a 100644 --- a/.github/workflows/python-eks-test.yml +++ b/.github/workflows/python-eks-test.yml @@ -64,8 +64,8 @@ jobs: - uses: actions/checkout@v4 with: - repository: 'aws-observability/aws-application-signals-test-framework' - ref: ${{ inputs.caller-workflow-name == 'main-build' && 'main' || github.ref }} + # repository: 'aws-observability/aws-application-signals-test-framework' + # ref: ${{ inputs.caller-workflow-name == 'main-build' && 'main' || github.ref }} fetch-depth: 0 # We initialize Gradlew Daemon early on during the workflow because sometimes initialization From 493fde2a0316c0480d2adf7b545f7584d209de0b Mon Sep 17 00:00:00 2001 From: Mingjie Shao Date: Tue, 24 Sep 2024 11:56:20 +0100 Subject: [PATCH 4/6] Adding a step to convert credential to base64 --- .github/workflows/python-eks-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-eks-test.yml b/.github/workflows/python-eks-test.yml index eaba6653a..1e490267b 100644 --- a/.github/workflows/python-eks-test.yml +++ b/.github/workflows/python-eks-test.yml @@ -208,6 +208,11 @@ jobs: RDS_MYSQL_CLUSTER_SECRETS, ${{env.RDS_MYSQL_CLUSTER_CREDENTIAL_SECRET_NAME}} parse-json-secrets: true + - name: Convert RDS database credentials to base64 + continue-on-error: true + run: | + echo "RDS_MYSQL_CLUSTER_SECRETS_PASSWORD_BASE64=$(echo -n '${{env.RDS_MYSQL_CLUSTER_SECRETS_PASSWORD}}' | base64)" >> $GITHUB_ENV + - name: Initiate Terraform uses: ./.github/workflows/actions/execute_and_retry with: @@ -246,7 +251,7 @@ jobs: -var='python_remote_app_image=${{ env.REMOTE_SAMPLE_APP_IMAGE_ARN }}' \ -var='rds_mysql_cluster_endpoint=${{env.RDS_MYSQL_CLUSTER_ENDPOINT}}' \ -var='rds_mysql_cluster_username=${{env.RDS_MYSQL_CLUSTER_SECRETS_USERNAME}}' \ - -var='rds_mysql_cluster_password=${{env.RDS_MYSQL_CLUSTER_SECRETS_PASSWORD}}' \ + -var='rds_mysql_cluster_password=${{env.RDS_MYSQL_CLUSTER_SECRETS_PASSWORD_BASE64}}' \ -var='rds_mysql_cluster_database=information_schema' \ -var='account_id=${{ env.ACCOUNT_ID }}' \ || deployment_failed=$? From 7806d8e468002c0257045c35155025beb5c06c01 Mon Sep 17 00:00:00 2001 From: Mingjie Shao Date: Tue, 24 Sep 2024 12:13:15 +0100 Subject: [PATCH 5/6] Update Python Sample App to decode password --- .../django_frontend_service/frontend_service_app/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sample-apps/python/django_frontend_service/frontend_service_app/views.py b/sample-apps/python/django_frontend_service/frontend_service_app/views.py index 336c3b2b4..c6556583f 100644 --- a/sample-apps/python/django_frontend_service/frontend_service_app/views.py +++ b/sample-apps/python/django_frontend_service/frontend_service_app/views.py @@ -2,6 +2,7 @@ ## SPDX-License-Identifier: Apache-2.0 import logging import os +import base64 import threading import time @@ -110,10 +111,14 @@ def get_xray_trace_id(): def mysql(request): logger.info("mysql received") + + encoded_password = os.environ["RDS_MYSQL_CLUSTER_PASSWORD"] + decoded_password = base64.b64decode(encoded_password).decode('utf-8') + try: connection = pymysql.connect(host=os.environ["RDS_MYSQL_CLUSTER_ENDPOINT"], user=os.environ["RDS_MYSQL_CLUSTER_USERNAME"], - password=os.environ["RDS_MYSQL_CLUSTER_PASSWORD"], + password=decoded_password, database=os.environ["RDS_MYSQL_CLUSTER_DATABASE"]) with connection: with connection.cursor() as cursor: From f0fd90498b653b3b0c0f462c3087e2052afaceb8 Mon Sep 17 00:00:00 2001 From: Mingjie Shao Date: Tue, 24 Sep 2024 14:14:49 +0100 Subject: [PATCH 6/6] Remove changes require for local testing --- .github/workflows/python-eks-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-eks-test.yml b/.github/workflows/python-eks-test.yml index 1e490267b..d769cce12 100644 --- a/.github/workflows/python-eks-test.yml +++ b/.github/workflows/python-eks-test.yml @@ -64,8 +64,8 @@ jobs: - uses: actions/checkout@v4 with: - # repository: 'aws-observability/aws-application-signals-test-framework' - # ref: ${{ inputs.caller-workflow-name == 'main-build' && 'main' || github.ref }} + repository: 'aws-observability/aws-application-signals-test-framework' + ref: ${{ inputs.caller-workflow-name == 'main-build' && 'main' || github.ref }} fetch-depth: 0 # We initialize Gradlew Daemon early on during the workflow because sometimes initialization