Fix Lambda canary trace validation failure (#345)

*Issue description:*
When searching the trace by API GetTraceSummaries, if the end time of
searching is set to be far away from the actual trace end time, xray
does not return Lambda traces. Right now I am not sure it is an aws sdk
xray client issue or xray service issue. This PR is a workaround by
moving trace validation step to be earlier, so the endTime of searching,
which the value is the current time, is close to the actual trace end
time. Then xray service can return the Lambda trace and pass the trace
validation.

*Description of changes:*

*Rollback procedure:*

<Can we safely revert this commit if needed? If not, detail what must be
done to safely revert and why it is needed.>

*Ensure you've run the following tests on your changes and include the
link below:*

To do so, create a `test.yml` file with `name: Test` and workflow
description to test your changes, then remove the file for your PR. Link
your test run in your PR description. This process is a short term
solution while we work on creating a staging environment for testing.

NOTE: TESTS RUNNING ON A SINGLE EKS CLUSTER CANNOT BE RUN IN PARALLEL.
See the
[needs](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idneeds)
keyword to run tests in succession.
- Run Java EKS on `e2e-playground` in us-east-1 and eu-central-2
- Run Python EKS on `e2e-playground` in us-east-1 and eu-central-2
- Run metric limiter on EKS cluster `e2e-playground` in us-east-1 and
eu-central-2
- Run EC2 tests in all regions
- Run K8s on a separate K8s cluster (check IAD test account for master
node endpoints; these will change as we create and destroy clusters for
OS patching)

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

Author: wangzlei

.github/workflows/node-lambda-test.yml

@@ -148,12 +148,11 @@ jobs:
         shell: bash
         run: sleep 30s; curl -sS ${{ env.API_GATEWAY_URL }}
 
-      # Validation for pulse telemetry data
-      - name: Validate generated EMF logs
-        id: log-validation
+      - name: Validate generated traces
+        id: trace-validation
         # will be removed after data quality bug fixed
         continue-on-error: true
-        run: ./gradlew validator:run --args='-c node/lambda/log-validation.yml
+        run: ./gradlew validator:run --args='-c node/lambda/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.API_GATEWAY_URL }}
           --region ${{ inputs.aws-region }}
@@ -163,12 +162,13 @@ jobs:
           --service-name ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}
           --rollup'
 
-      - name: Validate generated metrics
-        id: metric-validation
-        if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
+      # Validation for pulse telemetry data
+      - name: Validate generated EMF logs
+        id: log-validation
+        if: (success() || steps.trace-validation.outcome == 'failure') && !cancelled()
         # will be removed after data quality bug fixed
         continue-on-error: true
-        run: ./gradlew validator:run --args='-c node/lambda/metric-validation.yml
+        run: ./gradlew validator:run --args='-c node/lambda/log-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.API_GATEWAY_URL }}
           --region ${{ inputs.aws-region }}
@@ -178,12 +178,12 @@ jobs:
           --service-name ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}
           --rollup'
 
-      - name: Validate generated traces
-        id: trace-validation
-        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
+      - name: Validate generated metrics
+        id: metric-validation
+        if: (success() || steps.trace-validation.outcome == 'failure' || steps.log-validation.outcome == 'failure') && !cancelled()
         # will be removed after data quality bug fixed
         continue-on-error: true
-        run: ./gradlew validator:run --args='-c node/lambda/trace-validation.yml
+        run: ./gradlew validator:run --args='-c node/lambda/metric-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.API_GATEWAY_URL }}
           --region ${{ inputs.aws-region }}

.github/workflows/python-lambda-test.yml

@@ -174,12 +174,11 @@ jobs:
           max_retry: 3
           sleep_time: 60
 
-      # Validation for pulse telemetry data
-      - name: Validate generated EMF logs
-        id: log-validation
+      - name: Validate generated traces
+        id: trace-validation
         # will be removed after data quality bug fixed
         continue-on-error: true
-        run: ./gradlew validator:run --args='-c python/lambda/log-validation.yml
+        run: ./gradlew validator:run --args='-c python/lambda/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.API_GATEWAY_URL }}
           --region ${{ inputs.aws-region }}
@@ -189,12 +188,13 @@ jobs:
           --service-name ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}
           --rollup'
 
-      - name: Validate generated metrics
-        id: metric-validation
+      # Validation for pulse telemetry data
+      - name: Validate generated EMF logs
+        id: log-validation
+        if: (success() || steps.trace-validation.outcome == 'failure') && !cancelled()
         # will be removed after data quality bug fixed
         continue-on-error: true
-        if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
-        run: ./gradlew validator:run --args='-c python/lambda/metric-validation.yml
+        run: ./gradlew validator:run --args='-c python/lambda/log-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.API_GATEWAY_URL }}
           --region ${{ inputs.aws-region }}
@@ -204,12 +204,12 @@ jobs:
           --service-name ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}
           --rollup'
 
-      - name: Validate generated traces
-        id: trace-validation
+      - name: Validate generated metrics
+        id: metric-validation
         # will be removed after data quality bug fixed
         continue-on-error: true
-        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
-        run: ./gradlew validator:run --args='-c python/lambda/trace-validation.yml
+        if: (success() || steps.trace-validation.outcome == 'failure' || steps.log-validation.outcome == 'failure') && !cancelled()
+        run: ./gradlew validator:run --args='-c python/lambda/metric-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.API_GATEWAY_URL }}
           --region ${{ inputs.aws-region }}
@@ -246,4 +246,4 @@ jobs:
           -var="function_name=${{env.TERRAFORM_LAMBDA_FUNCTION_NAME}}" \
           -var="layer_artifacts_directory=${{ env.ARTIFACTS_DIR }}" \
           -var="region=${{ env.E2E_TEST_AWS_REGION }}" \
-          -var="is_canary=${{ env.IS_CANARY }}"
+          -var="is_canary=${{ env.IS_CANARY }}"

validator/src/main/java/com/amazon/aoc/enums/GenericConstants.java

@@ -20,7 +20,7 @@
 @Getter
 public enum GenericConstants {
   // retry
-  SLEEP_IN_MILLISECONDS("30000"), // ms
+  SLEEP_IN_MILLISECONDS("10000"), // ms
   SLEEP_IN_SECONDS("30"),
   MAX_RETRIES("10"),
 

validator/src/main/java/com/amazon/aoc/services/XRayService.java

@@ -25,13 +25,11 @@
 import com.amazonaws.services.xray.model.TraceSummary;
 import java.util.Date;
 import java.util.List;
-import lombok.extern.log4j.Log4j2;
 import org.joda.time.DateTime;
 
-@Log4j2
 public class XRayService {
   private AWSXRay awsxRay;
-  private final int SEARCH_PERIOD = 600 * 3;
+  private final int SEARCH_PERIOD = 600;
   public static String DEFAULT_TRACE_ID = "1-00000000-000000000000000000000000";
 
   public XRayService(String region) {
@@ -55,7 +53,6 @@ public List<Trace> listTraceByIds(List<String> traceIdList) {
   public List<TraceSummary> searchTraces(String traceFilter) {
     Date currentDate = new Date();
     Date pastDate = new DateTime(currentDate).minusSeconds(SEARCH_PERIOD).toDate();
-    log.info("--start-time: " + pastDate + ", --end-time: " + currentDate + ", traceFilter: " + traceFilter);
     GetTraceSummariesResult traceSummaryResult =
             awsxRay.getTraceSummaries(
                     new GetTraceSummariesRequest()