BCDA-9671: Update deploy-all worker version check (#1286)

## 🎫 Ticket

https://jira.cms.gov/browse/BCDA-9671

## 🛠 Changes

<!-- What was added, updated, or removed in this PR? -->

- re-enable the worker version check in the deploy-all workflow
- update the worker version check to check the version of the tag used
by the worker service

## ℹ️ Context

<!-- Why were these changes made? Add background context suitable for a
non-technical audience. -->

There are checks for each service that ensure the correct version of the
service has been deployed. The api and ssas services have endpoints that
provide this info, but the worker version must be fetched another way.
This PR intends to re-enable this version check and update it to grab
the version from the image associated with the service's running task.

<!-- If any of the following security implications apply, this PR must
not be merged without Stephen Walter's approval. Explain in this section
and add @SJWalter11 as a reviewer.
  - Adds a new software dependency or dependencies.
  - Modifies or invalidates one or more of our security controls.
  - Stores or transmits data that was not stored or transmitted before.
- Requires additional review of security implications for other reasons.
-->

## 🧪 Validation

<!-- How were the changes verified? Did you fully test the acceptance
criteria in the ticket? Provide reproducible testing instructions and
screenshots if applicable. -->

Tested with [deployment to
dev](https://github.com/CMSgov/bcda-app/actions/runs/20601880004)

Author: michaeljvaldes

.github/workflows/deploy-all.yml

@@ -91,8 +91,6 @@ env:
   ENV_MODIFIER: ${{ inputs.env || 'dev' }}
   TEST_ACO: ${{ inputs.test_aco || 'dev' }}
   TENV_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  VERIFICATION_RETRIES: 90 # 90 retries with 10s sleep = max 900s or 15m.  Verification jobs run in parallel.
-  VERIFICATION_SLEEP: 10
 
 jobs:
   migrate_db:
@@ -155,9 +153,9 @@ jobs:
       # FARGATE FIXUP: Can remove this step
       - name: Get AMIs
         run: |
-          export BCDA_AMI=`aws ec2 describe-images --region ${{ vars.AWS_REGION }} --filters 'Name=tag:app,Values=bcda-app' 'Name=tag:version,Values=${{ env.RELEASE_VERSION }}' --query 'Images[*][CreationDate,ImageId] | reverse(sort_by(@,&[0])) | [0][1]' --output text`
+          export BCDA_AMI=$(aws ec2 describe-images --region ${{ vars.AWS_REGION }} --filters 'Name=tag:app,Values=bcda-app' 'Name=tag:version,Values=${{ env.RELEASE_VERSION }}' --query 'Images[*][CreationDate,ImageId] | reverse(sort_by(@,&[0])) | [0][1]' --output text)
           echo "BCDA_AMI=$BCDA_AMI" >> $GITHUB_ENV
-          export WORKER_AMI=`aws ec2 describe-images --region ${{ vars.AWS_REGION }} --filters 'Name=tag:app,Values=bcda-worker' 'Name=tag:version,Values=${{ env.RELEASE_VERSION }}' --query 'Images[*][CreationDate,ImageId] | reverse(sort_by(@,&[0])) | [0][1]' --output text`
+          export WORKER_AMI=$(aws ec2 describe-images --region ${{ vars.AWS_REGION }} --filters 'Name=tag:app,Values=bcda-worker' 'Name=tag:version,Values=${{ env.RELEASE_VERSION }}' --query 'Images[*][CreationDate,ImageId] | reverse(sort_by(@,&[0])) | [0][1]' --output text)
           echo "WORKER_AMI=$WORKER_AMI" >> $GITHUB_ENV
       - name: Install Cosign to verify tenv and tofu installs
         uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2
@@ -199,84 +197,56 @@ jobs:
     runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
     environment: ${{ inputs.env != '0' && inputs.env || 'dev' }}
     steps:
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/delegatedadmin/developer/${{ vars.AWS_ROLE_TO_ASSUME }}
       - run: |
-          TRY=1
-
-          until [ $TRY -gt $VERIFICATION_RETRIES ]; do
-            BCDA_API_VERSION=`curl https://${{ vars.API_BASE_URL }}/_version | jq -R '. as $line | try (fromjson | .version) catch $line' | tr -d "\"'"`
-
-            if [[ $BCDA_API_VERSION != ${{ env.RELEASE_VERSION }} ]]; then
-              echo "BCDA API expected release version: ${{ env.RELEASE_VERSION }} did not match curled version: ${BCDA_API_VERSION}."
-              TRY=$(($TRY + 1))
-              if [ $TRY -gt $VERIFICATION_RETRIES ]; then
-                exit 1
-              else
-                sleep $VERIFICATION_SLEEP
-              fi
-            else
-              break
-            fi
-          done
+          aws ecs wait services-stable --cluster bcda-${{ env.RELEASE_ENV }} --services bcda-${{ env.RELEASE_ENV }}-api
+          BCDA_API_VERSION=$(curl -Ss https://${{ vars.API_BASE_URL }}/_version | jq -R '. as $line | try (fromjson | .version) catch $line' | tr -d "\"'")
+          if [[ $BCDA_API_VERSION != ${{ env.RELEASE_VERSION }} ]]; then
+            echo "BCDA API expected release version: ${{ env.RELEASE_VERSION }} did not match curled version: ${BCDA_API_VERSION}."
+            exit 1
+          fi
 
   verify_ssas_version:
     needs: [deploy]
     runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
     environment: ${{inputs.env != '0' && inputs.env || 'dev' }}
     steps:
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/delegatedadmin/developer/${{ vars.AWS_ROLE_TO_ASSUME }}
       - run: |
-          TRY=1
-
-          until [ $TRY -gt $VERIFICATION_RETRIES ]; do
-            BCDA_SSAS_VERSION=`curl https://${{ vars.API_BASE_URL }}/_auth | jq -R '. as $line | try (fromjson | .version) catch $line' | tr -d "\"'"`
-
-            if [[ $BCDA_SSAS_VERSION != ${{ env.SSAS_RELEASE_VERSION }} ]]; then
-              echo "BCDA SSAS expected release version: ${{ env.SSAS_RELEASE_VERSION }} did not match curled version: ${BCDA_SSAS_VERSION}."
-              TRY=$(($TRY + 1))
-              if [ $TRY -gt $VERIFICATION_RETRIES ]; then
-                exit 1
-              else
-                sleep $VERIFICATION_SLEEP
-              fi
-            else
-              break
-            fi
-          done
+          aws ecs wait services-stable --cluster bcda-${{ env.RELEASE_ENV }} --services bcda-${{ env.RELEASE_ENV }}-ssas
+          BCDA_SSAS_VERSION=$(curl -Ss https://${{ vars.API_BASE_URL }}/_auth | jq -R '. as $line | try (fromjson | .version) catch $line' | tr -d "\"'")
+          if [[ $BCDA_SSAS_VERSION != ${{ env.SSAS_RELEASE_VERSION }} ]]; then
+            echo "BCDA SSAS expected release version: ${{ env.SSAS_RELEASE_VERSION }} did not match curled version: ${BCDA_SSAS_VERSION}."
+            exit 1
+          fi
 
-  # Temporarily we need to manually verify worker is on correct version
-  
-  # # FARGATE FIXUP:
-  # This needs the 'image_tag' tag to be added to each service before this will work
-  # verify_worker_version:
-  #   needs: [deploy]
-  #   runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
-  #   environment: ${{ inputs.env != '0' && inputs.env || 'dev' }}
-  #   steps:
-  #     - uses: aws-actions/configure-aws-credentials@v4
-  #       with:
-  #         aws-region: ${{ vars.AWS_REGION }}
-  #         role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/delegatedadmin/developer/${{ vars.AWS_ROLE_TO_ASSUME }}
-  #     - run: |
-  #         TRY=1
-
-  #         until [ $TRY -gt $VERIFICATION_RETRIES ]; do
-  #           WORKER_VERSION=`aws ecs --resource-arn list-tags-for-resource arn:aws:ecs:${{ vars.AWS_REGION }}:${{ secrets.AWS_ACCOUNT_ID }}:service/bcda-${{ env.RELEASE_ENV }}/bcda-${{ env.RELEASE_ENV }}-api --query "tags[?key=='image_tag'].value | [0]" --output text`
-
-  #           if [[ $WORKER_VERSION != ${{ env.RELEASE_VERSION }} ]]; then
-  #             echo "BCDA Worker expected release version: ${{ env.RELEASE_VERSION }} did not match AWS version: ${WORKER_VERSION}."
-  #             TRY=$(($TRY + 1))
-  #             if [ $TRY -gt $VERIFICATION_RETRIES ]; then
-  #               exit 1
-  #             else
-  #               sleep $VERIFICATION_SLEEP
-  #             fi
-  #           else
-  #             break
-  #           fi
-  #         done
+  verify_worker_version:
+    needs: [deploy]
+    runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
+    environment: ${{ inputs.env != '0' && inputs.env || 'dev' }}
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/delegatedadmin/developer/${{ vars.AWS_ROLE_TO_ASSUME }}
+      - run: |
+          aws ecs wait services-stable --cluster bcda-${{ env.RELEASE_ENV }} --services bcda-${{ env.RELEASE_ENV }}-worker
+          WORKER_ARN=$(aws ecs list-tasks --cluster=bcda-${{ env.RELEASE_ENV }} --service-name=bcda-${{ env.RELEASE_ENV }}-worker --query 'taskArns[0]' --output text)
+          WORKER_IMAGE=$(aws ecs describe-tasks --cluster=bcda-${{ env.RELEASE_ENV }} --tasks=$WORKER_ARN --query="tasks[0].containers[0].image" --output=text)
+          WORKER_VERSION=${WORKER_IMAGE#*:}
+          if [[ $WORKER_VERSION != ${{ env.RELEASE_VERSION }} ]]; then
+            echo "BCDA Worker expected release version: ${{ env.RELEASE_VERSION }} did not match AWS version: ${WORKER_VERSION}."
+            exit 1
+          fi
 
   smoketests:
-    # needs: [migrate_db, migrate_ssas_db, deploy, verify_api_version, verify_ssas_version, verify_worker_version]
-    needs: [migrate_db, migrate_ssas_db, deploy, verify_api_version, verify_ssas_version]
+    needs: [migrate_db, migrate_ssas_db, deploy, verify_api_version, verify_ssas_version, verify_worker_version]
     uses: ./.github/workflows/smoke-tests.yml
     with:
       release_version: ${{ inputs.release_version || 'main' }}
@@ -290,8 +260,7 @@ jobs:
     secrets: inherit
 
   notify_newrelic:
-    # needs: [migrate_db, migrate_ssas_db, deploy, verify_api_version, verify_ssas_version, verify_worker_version]
-    needs: [migrate_db, migrate_ssas_db, deploy, verify_api_version, verify_ssas_version]
+    needs: [migrate_db, migrate_ssas_db, deploy, verify_api_version, verify_ssas_version, verify_worker_version]
     environment: ${{ inputs.env || 'dev' }}
     runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
     steps:
@@ -322,7 +291,7 @@ jobs:
       - name: Notify NewRelic (Dev)
         if: ${{ env.RELEASE_ENV == 'dev' }}
         run: |
-          export BCDA_AMI=`aws ec2 describe-images --region ${{ vars.AWS_REGION }} --filters 'Name=tag:app,Values=bcda-app' 'Name=tag:version,Values=${{ env.RELEASE_VERSION }}' --query 'Images[*][CreationDate,ImageId] | reverse(sort_by(@,&[0])) | [0][1]' --output text`
+          export BCDA_AMI=$(aws ec2 describe-images --region ${{ vars.AWS_REGION }} --filters 'Name=tag:app,Values=bcda-app' 'Name=tag:version,Values=${{ env.RELEASE_VERSION }}' --query 'Images[*][CreationDate,ImageId] | reverse(sort_by(@,&[0])) | [0][1]' --output text)
           python3 scripts/mark_deployment.py \
             --app_id ${{ env.NEWRELIC_APP_ID }} \
             --api_key ${{ env.NEWRELIC_API_KEY }} \