Apply retry using enablement script (#146)

zzhlogin · web-flow · commit 28976d1f0d72 · 2024-04-10T11:12:58.000-07:00
*Issue #, if available:* Apply retry using enablement script to align with Java E2E tests. sample workflow tests: EKS E2E:https://github.com/aws-observability/aws-otel-python-instrumentation/actions/runs/8622097463 EC2 E2E:https://github.com/aws-observability/aws-otel-python-instrumentation/actions/runs/8622052335 By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
diff --git a/.github/workflows/appsignals-python-e2e-ec2-test.yml b/.github/workflows/appsignals-python-e2e-ec2-test.yml
@@ -32,7 +32,7 @@ env:
   METRIC_NAMESPACE: AppSignals
   LOG_GROUP_NAME: /aws/appsignals/generic
   ADOT_WHEEL_NAME: ${{ inputs.staging_wheel_name }}
-
+  TEST_RESOURCES_FOLDER: /home/runner/work/aws-application-signals-test-framework/aws-application-signals-test-framework
 
 jobs:
   python-e2e-ec2-test:
@@ -88,15 +88,21 @@ jobs:
           fi
 
       - name: Set up terraform
-        uses: hashicorp/setup-terraform@v3
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg"
+          post-command: 'echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          && sudo apt update && sudo apt install terraform'
+
+      - name: Initiate Terraform
+        uses: ./.github/workflows/actions/execute_and_retry
         with:
-          terraform_wrapper: false
+          command: "cd terraform/python/ec2 && terraform init && terraform validate"
+          cleanup: "rm -rf .terraform && rm -rf .terraform.lock.hcl"
 
       - name: Deploy sample app via terraform and wait for endpoint to come online
         working-directory: terraform/python/ec2
         run: |
-          terraform init
-          terraform validate
           # Attempt to deploy the sample app on an EC2 instance and wait for its endpoint to come online. 
           # There may be occasional failures due to transitivity issues, so try up to 2 times. 
           # deployment_failed of 0 indicates that both the terraform deployment and the endpoint are running, while 1 indicates
diff --git a/.github/workflows/appsignals-python-e2e-eks-test.yml b/.github/workflows/appsignals-python-e2e-eks-test.yml
@@ -35,6 +35,7 @@ env:
   PYTHON_SAMPLE_APP_NAMESPACE: python-app-namespace
   METRIC_NAMESPACE: AppSignals
   LOG_GROUP_NAME: /aws/appsignals/eks
+  TEST_RESOURCES_FOLDER: /home/runner/work/aws-application-signals-test-framework/aws-application-signals-test-framework
 
 jobs:
   python-e2e-eks-test:
@@ -47,24 +48,17 @@ jobs:
           fetch-depth: 0
 
       - name: Download enablement script
-        uses: actions/checkout@v4
+        uses: ./.github/workflows/actions/execute_and_retry
         with:
-          repository: aws-observability/application-signals-demo
-          ref: main
-          path: enablement-script
-          sparse-checkout: |
-            scripts/eks/appsignals/enable-app-signals.sh
-            scripts/eks/appsignals/clean-app-signals.sh
-          sparse-checkout-cone-mode: false
-
-      - name: Resolve Add-on configuration conflict
-        working-directory: enablement-script/scripts/eks/appsignals
-        run: |
-          sed -i 's/aws eks create-addon \\/aws eks create-addon \\\n        --resolve-conflicts OVERWRITE \\/' enable-app-signals.sh
+          pre-command: "mkdir enablement-script && cd enablement-script"
+          command: "wget https://raw.githubusercontent.com/aws-observability/application-signals-demo/main/scripts/eks/appsignals/enable-app-signals.sh 
+          && wget https://raw.githubusercontent.com/aws-observability/application-signals-demo/main/scripts/eks/appsignals/clean-app-signals.sh"
+          cleanup: "rm -f enable-app-signals.sh && rm -f clean-app-signals.sh"
+          post-command: "chmod +x enable-app-signals.sh && chmod +x clean-app-signals.sh"
 
       - name: Remove log group deletion command
         if: always()
-        working-directory: enablement-script/scripts/eks/appsignals
+        working-directory: enablement-script
         run: |
           delete_log_group="aws logs delete-log-group --log-group-name '${{ env.LOG_GROUP_NAME }}' --region \$REGION"
           sed -i "s#$delete_log_group##g" clean-app-signals.sh
@@ -103,38 +97,47 @@ jobs:
         run: aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}
 
       - name: Install eksctl
-        working-directory: .github/
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          pre-command: 'mkdir ${{ github.workspace }}/eksctl'
+          command: 'curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz" 
+                  && tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz'
+          cleanup: 'rm -f eksctl_Linux_amd64.tar.gz'
+
+      - name: Add eksctl to Github Path
         run: |
-          source ./workflows/util/execute_and_retry.sh
-          mkdir ${{ github.workspace }}/eksctl
-          curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz"
-          execute_and_retry 2 "tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz"
           echo "${{ github.workspace }}/eksctl" >> $GITHUB_PATH
 
       - name: Create role for AWS access from the sample app
         id: create_service_account
-        run: |
-          eksctl create iamserviceaccount \
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "eksctl create iamserviceaccount \
           --name service-account-${{ env.TESTING_ID }} \
           --namespace ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }} \
           --cluster ${{ inputs.test-cluster-name }} \
           --role-name eks-s3-access-${{ env.TESTING_ID }} \
           --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \
           --region ${{ inputs.aws-region }} \
-          --approve
+          --approve"
 
       - name: Set up terraform
-        uses: hashicorp/setup-terraform@v3
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg"
+          post-command: 'echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          && sudo apt update && sudo apt install terraform'
+
+      - name: Initiate Terraform
+        uses: ./.github/workflows/actions/execute_and_retry
         with:
-          terraform_wrapper: false
+          command: "cd terraform/python/eks && terraform init && terraform validate"
+          cleanup: "rm -rf .terraform && rm -rf .terraform.lock.hcl"
 
       - name: Deploy sample app via terraform and wait for the endpoint to come online
         id: deploy-python-app
         working-directory: terraform/python/eks
         run: |
-          terraform init
-          terraform validate
-          
           # Attempt to deploy the sample app on an EKS instance and wait for its endpoint to come online. 
           # There may be occasional failures due to transitivity issues, so try up to 2 times. 
           # deployment_failed of 0 indicates that both the terraform deployment and the endpoint are running, while 1 indicates
@@ -163,26 +166,23 @@ jobs:
             # If the deployment_failed is still 0, then the terraform deployment succeeded and now try to connect to the endpoint 
             # after installing App Signals. Attempts to connect will be made for up to 10 minutes
             if [ $deployment_failed -eq 0 ]; then          
-              kubectl wait --for=condition=Ready pod --all -n ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }}
-
               echo "Installing app signals to the sample app"
               source ${GITHUB_WORKSPACE}/.github/workflows/util/execute_and_retry.sh
               execute_and_retry 2 \
-              "${GITHUB_WORKSPACE}/enablement-script/scripts/eks/appsignals/enable-app-signals.sh \
+              "${GITHUB_WORKSPACE}/enablement-script/enable-app-signals.sh \
               ${{ inputs.test-cluster-name }} \
               ${{ inputs.aws-region }} \
               ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }}" \
-              "${GITHUB_WORKSPACE}/enablement-script/scripts/eks/appsignals/clean-app-signals.sh \
+              "${GITHUB_WORKSPACE}/enablement-script/clean-app-signals.sh \
               ${{ inputs.test-cluster-name }} \
               ${{ inputs.aws-region }} \
               ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }} && \
               aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}"
 
-
               execute_and_retry 2 "kubectl delete pods --all -n ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }}"
               execute_and_retry 2 "kubectl wait --for=condition=Ready --request-timeout '5m' pod --all -n ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }}"
 
-              echo "Attempting to connect to the endpoint"
+              echo "Attempting to connect to the main sample app endpoint"
               python_app_endpoint=http://$(terraform output python_app_endpoint)
               attempt_counter=0
               max_attempts=60
@@ -203,9 +203,13 @@ jobs:
             # resources created from terraform and try again.
             if [ $deployment_failed -eq 1 ]; then
               echo "Cleaning up App Signal"
+              ${GITHUB_WORKSPACE}/enablement-script/clean-app-signals.sh \
+              ${{ inputs.test-cluster-name }} \
+              ${{ inputs.aws-region }} \
+              ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }}
 
-              cd ${{ github.workspace }}/amazon-cloudwatch-agent-operator/
-              kubectl delete -f ./namespace.yaml
+              # Running clean-app-signal.sh removes the current cluster from the config. Update the cluster again for subsequent runs.
+              aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}
 
               echo "Destroying terraform"
               terraform destroy -auto-approve \
@@ -231,20 +235,6 @@ jobs:
             fi
           done
 
-          # Attach policies to cluster node group roles that are required for AppSignals
-          aws eks list-nodegroups --cluster-name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }} |\
-          jq -r '.nodegroups[]' |\
-          while read -r node_group;
-          do
-            node_role=$(\
-              aws eks describe-nodegroup  --cluster-name ${{ inputs.test-cluster-name }} --nodegroup-name $node_group --region ${{ inputs.aws-region }} |\
-              jq -r '.nodegroup.nodeRole' |\
-              cut -d'/' -f2
-            )
-            aws iam attach-role-policy --role-name $node_role --policy-arn arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy --region ${{ inputs.aws-region }}
-            aws iam attach-role-policy --role-name $node_role --policy-arn arn:aws:iam::aws:policy/AWSXRayWriteOnlyAccess --region ${{ inputs.aws-region }}
-          done
-
       - name: Get remote service pod name and IP
         run: |
           echo "REMOTE_SERVICE_DEPLOYMENT_NAME=$(kubectl get deployments -n ${{ env.PYTHON_SAMPLE_APP_NAMESPACE }} --selector=app=remote-app -o jsonpath='{.items[0].metadata.name}')" >> $GITHUB_ENV
@@ -348,7 +338,7 @@ jobs:
       - name: Clean Up App Signals
         if: always()
         continue-on-error: true
-        working-directory: enablement-script/scripts/eks/appsignals
+        working-directory: enablement-script
         run: |
           ./clean-app-signals.sh \
           ${{ inputs.test-cluster-name }} \
