BCDA-9397: Make sure lambda workflow verify steps run on call (#1202)

## 🎫 Ticket

https://jira.cms.gov/browse/BCDA-9397

## 🛠 Changes

Adjust cclf import and opt-out lambda integration test workflows to make
sure the verify process works when called via on_dispatch.

## ℹ️ Context

Recent manual runs of both have skipped the verify step:
https://github.com/CMSgov/bcda-app/actions/runs/17273221595

<!-- If any of the following security implications apply, this PR must
not be merged without Stephen Walter's approval. Explain in this section
and add @SJWalter11 as a reviewer.
  - Adds a new software dependency or dependencies.
  - Modifies or invalidates one or more of our security controls.
  - Stores or transmits data that was not stored or transmitted before.
- Requires additional review of security implications for other reasons.
-->

## 🧪 Validation

Workflow run on push: 
-
cclfhttps://github.com/CMSgov/bcda-app/actions/runs/17274630077/job/49028241943?pr=1202
(cclf import)

Author: carlpartridge

.dockerignore

@@ -11,3 +11,5 @@
 **/test_results
 !./bcdaworker/data/test/test.ndjson
 **/bcda/pending_delete_dir/*
+
+**/.DS_Store

.github/workflows/cclf-import-integration-test.yml

@@ -35,9 +35,6 @@ jobs:
     defaults:
       run:
         working-directory: bcda
-    outputs:
-      cclffilename: ${{ steps.createfile.outputs.cclffilename }}
-      csvfilename: ${{ steps.createfile.outputs.csvfilename }}
     steps:
       - uses: actions/checkout@v4
       - uses: aws-actions/configure-aws-credentials@v4
@@ -59,21 +56,20 @@ jobs:
           role-chaining: true
           role-skip-session-tagging: true
       - name: Upload test file to the BFD bucket to trigger lambda function via SNS message
-        id: createfile
         run: |
           year=$(date +'%y')
           date=$(date +'%y%m%d')
           time=$(date +'%H%M%S')
 
           fname=T.BCD.A0001.ZCY${year}.D${date}.T${time}1
           cclf8_fname=T.BCD.A0001.ZC8Y${year}.D${date}.T${time}1
-          echo "CCLFFILENAME=$cclf8_fname" >> "$GITHUB_OUTPUT"
+          echo "CCLFFILENAME=$cclf8_fname" >> "$GITHUB_ENV"
 
           csvname=T.PCPB.M${year}11.D${date}.T${time}1
-          echo "CSVFILENAME=$csvname" >> "$GITHUB_OUTPUT"
+          echo "CSVFILENAME=$csvname" >> "$GITHUB_ENV"
 
           guidecsvname=T.GUIDE.GUIDE-00001.Y${year}.D${date}.T${time}1
-          echo "GUIDEFILENAME=$guidecsvname" >> "$GITHUB_OUTPUT"
+          echo "GUIDEFILENAME=$guidecsvname" >> "$GITHUB_ENV"
 
           mv ../shared_files/cclf/archives/csv/P.PCPB.M2411.D181120.T1000000 ${csvname}
 
@@ -97,16 +93,8 @@ jobs:
 
           aws s3 cp --no-progress ${guidecsvname} \
             s3://bfd-test-eft/bfdeft01/bcda/in/test/${guidecsvname}
-
-            
-
-  verify:
-    needs: trigger
-    runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true"
-    steps:
-      - uses: actions/checkout@v3
+      # Give a bit of time for lambdas to process files
+      - run: sleep 60
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ vars.AWS_REGION }}
@@ -124,17 +112,11 @@ jobs:
             CONNECTION_INFO=/bcda/test/api/DATABASE_URL
       - name: Verify CCLF file was ingested
         env:
-          CCLFFILENAME: ${{ needs.trigger.outputs.cclffilename }}
-          CSVFILENAME: ${{ needs.trigger.outputs.csvfilename }}
-          GUIDEFILENAME: ${{ needs.trigger.outputs.guidefilename }}
           PGSSLMODE: require
         # CAUTION: if changing the script below, validate that sensitive information is not printed in the workflow
         run: |
           HOST=$(aws rds describe-db-cluster-endpoints --db-cluster-identifier bcda-test-aurora 2>&1 | jq -r '.DBClusterEndpoints[0].Endpoint' 2>&1)
-          echo $HOST | cut -c -10
           CONNECTION_URL=$(echo $CONNECTION_INFO 2>&1 | sed -E "s/@.*\/bcda/\@$HOST\/bcda/" 2>&1)
-          echo $CONNECTION_URL | cut -c -5
-          echo $CCLFFILENAME
 
           # Verify that we have a record of the CCLF file in the database
           CCLF_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CCLFFILENAME' LIMIT 1" 2>&1`

.github/workflows/opt-out-import-integration-test.yml

@@ -36,22 +36,19 @@ jobs:
     defaults:
       run:
         working-directory: ./optout
-    outputs:
-      filename: ${{ steps.createfile.outputs.FILENAME }}
     steps:
       - uses: actions/checkout@v4
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ vars.AWS_REGION }}
-          role-to-assume: arn:aws:iam::${{ secrets.NON_PROD_ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-opt-out-export-function
+          role-to-assume: arn:aws:iam::${{ secrets.NON_PROD_ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-opt-out-import-function
       - name: Get BFD Account
         uses: cmsgov/cdap/actions/aws-params-env-action@main
         env:
           AWS_REGION: ${{ vars.AWS_REGION }}
         with:
           params: | 
             BFD_ACCOUNT_ID=/bfd/account-id
-      
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ vars.AWS_REGION }}
@@ -60,20 +57,13 @@ jobs:
           role-chaining: true
           role-skip-session-tagging: true
       - name: Upload test file to the BFD bucket to trigger lambda function via SNS message
-        id: createfile
         run: |
           fname=T\#EFT.ON.ACO.NGD1800.DPRF.D$(date +'%y%m%d').T$(date +'%H%M%S')1
-          echo "FILENAME=$fname" >> "$GITHUB_OUTPUT"
-          aws s3 cp ../shared_files/synthetic1800MedicareFiles/test/T\#EFT.ON.ACO.NGD1800.DPRF.D181120.T1000009 \
+          echo "FILENAME=$fname" >> "$GITHUB_ENV"
+          aws s3 cp --no-progress ../shared_files/synthetic1800MedicareFiles/test/T\#EFT.ON.ACO.NGD1800.DPRF.D181120.T1000009 \
             s3://bfd-test-eft/bfdeft01/bcda/in/test/$fname
-
-  verify:
-    needs: trigger
-    runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true"
-    steps:
-      - uses: actions/checkout@v3
+      # Give a bit of time for lambdas to process files
+      - run: sleep 60
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ vars.AWS_REGION }}
@@ -86,18 +76,15 @@ jobs:
         uses: cmsgov/cdap/actions/aws-params-env-action@main
         env:
           AWS_REGION: ${{ vars.AWS_REGION }}
-
         with:
           params: |
             CONNECTION_INFO=/bcda/test/api/DATABASE_URL
       - name: Verify suppression file was ingested
-        env:
-          FILENAME: ${{needs.trigger.outputs.filename}}
         # CAUTION: if changing the script below, validate that sensitive information is not printed in the workflow
         run: |
           HOST=$(aws rds describe-db-cluster-endpoints --db-cluster-identifier bcda-test-aurora 2>&1 | jq -r '.DBClusterEndpoints[0].Endpoint' 2>&1)
           CONNECTION_URL=$(echo $CONNECTION_INFO 2>&1 | sed -E "s/@.*\/bcda/\@$HOST\/bcda/" 2>&1)
-          echo $FILENAME
+
           SUPPRESSION_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM suppression_files WHERE name = '$FILENAME' LIMIT 1" 2>&1`
           if [[ $? -ne 0 || -z $SUPPRESSION_FILE ]]; then
             echo "suppression_file query returned zero results or command failed"
@@ -109,4 +96,3 @@ jobs:
                 exit 1
               fi
           fi
-