BCDA-9484: Migrate aws sdk to v2 #137
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Integration test that triggers a CCLF import in the test environment and | |
| # verifies the imported data in the test database for ACO A0001. | |
| name: cclf-import test integration | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/cclf-import-integration-test.yml | |
| - .github/workflows/cclf-import-deploy.yml | |
| - bcda/cclf/** | |
| - bcda/lambda/cclf/** | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| id-token: write | |
| # Ensure we have only one integration test running at a time | |
| concurrency: | |
| group: cclf-import-test-integration | |
| jobs: | |
| # Deploy first if triggered by pull_request | |
| deploy: | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: ./.github/workflows/cclf-import-deploy.yml | |
| secrets: inherit | |
| with: | |
| deploy_env: test | |
| trigger: | |
| if: ${{ always() }} | |
| needs: deploy | |
| runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}} | |
| defaults: | |
| run: | |
| working-directory: bcda | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ vars.AWS_REGION }} | |
| role-to-assume: arn:aws:iam::${{ secrets.NON_PROD_ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-cclf-import-function | |
| - name: Get BFD Account | |
| uses: cmsgov/cdap/actions/aws-params-env-action@main | |
| env: | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| with: | |
| params: | | |
| BFD_ACCOUNT_ID=/bfd/account-id | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ vars.AWS_REGION }} | |
| # Note that we use the BFD role with access to the bucket | |
| role-to-assume: arn:aws:iam::${{ env.BFD_ACCOUNT_ID }}:role/delegatedadmin/developer/bfd-test-eft-bcda-ct-bucket-role | |
| role-chaining: true | |
| role-skip-session-tagging: true | |
| - name: Upload test file to the BFD bucket to trigger lambda function via SNS message | |
| run: | | |
| year=$(date +'%y') | |
| date=$(date +'%y%m%d') | |
| time=$(date +'%H%M%S') | |
| fname=T.BCD.A0001.ZCY${year}.D${date}.T${time}1 | |
| cclf8_fname=T.BCD.A0001.ZC8Y${year}.D${date}.T${time}1 | |
| echo "CCLFFILENAME=$cclf8_fname" >> "$GITHUB_ENV" | |
| csvname=T.PCPB.M${year}11.D${date}.T${time}1 | |
| echo "CSVFILENAME=$csvname" >> "$GITHUB_ENV" | |
| guidecsvname=T.GUIDE.GUIDE-00001.Y${year}.D${date}.T${time}1 | |
| echo "GUIDEFILENAME=$guidecsvname" >> "$GITHUB_ENV" | |
| mv ../shared_files/cclf/archives/csv/P.PCPB.M2411.D181120.T1000000 ${csvname} | |
| mv ../shared_files/cclf/archives/csv/T.GUIDE.GUIDE-00001.Y25.D250630.T1000000 ${guidecsvname} | |
| unzip ../shared_files/cclf/archives/valid/T.BCD.A0001.ZCY18.D181120.T1000000 | |
| # Clone the synthetic zip file with updated years | |
| new_dir="D${date}.T${time}" | |
| mkdir $new_dir | |
| mv T.BCD.A0001.ZC8Y18.D181120.T1000009 $new_dir/T.BCD.A0001.ZC8Y${year}.D${date}.T${time}1 | |
| mv T.BCD.A0001.ZC9Y18.D181120.T1000010 $new_dir/T.BCD.A0001.ZC9Y${year}.D${date}.T${time}1 | |
| mv T.BCD.A0001.ZC0Y18.D181120.T1000011 $new_dir/T.BCD.A0001.ZC0Y${year}.D${date}.T${time}1 | |
| zip -jr $fname $new_dir | |
| aws s3 cp --no-progress $fname \ | |
| s3://bfd-test-eft/bfdeft01/bcda/in/test/$fname | |
| aws s3 cp --no-progress ${csvname} \ | |
| s3://bfd-test-eft/bfdeft01/bcda/in/test/${csvname} | |
| aws s3 cp --no-progress ${guidecsvname} \ | |
| s3://bfd-test-eft/bfdeft01/bcda/in/test/${guidecsvname} | |
| # Give a bit of time for lambdas to process files | |
| - run: sleep 60 | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ vars.AWS_REGION }} | |
| role-to-assume: arn:aws:iam::${{ secrets.NON_PROD_ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-github-actions | |
| - name: Install psql | |
| run: | | |
| sudo dnf -y update | |
| sudo dnf install postgresql16 -y | |
| - name: Get database credentials | |
| uses: cmsgov/cdap/actions/aws-params-env-action@main | |
| env: | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| with: | |
| params: | | |
| CONNECTION_INFO=/bcda/test/api/DATABASE_URL | |
| - name: Verify CCLF file was ingested | |
| env: | |
| PGSSLMODE: require | |
| # CAUTION: if changing the script below, validate that sensitive information is not printed in the workflow | |
| run: | | |
| HOST=$(aws rds describe-db-cluster-endpoints --db-cluster-identifier bcda-test-aurora 2>&1 | jq -r '.DBClusterEndpoints[0].Endpoint' 2>&1) | |
| CONNECTION_URL=$(echo $CONNECTION_INFO 2>&1 | sed -E "s/@.*\/bcda/\@$HOST\/bcda/" 2>&1) | |
| # Verify that we have a record of the CCLF file in the database | |
| CCLF_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CCLFFILENAME' LIMIT 1" 2>&1` | |
| echo "CCLF_FILE: $CCLF_FILE" | |
| if [[ $? -ne 0 || -z $CCLF_FILE ]]; then | |
| echo "cclf_file query returned zero results or command failed" | |
| exit 1 | |
| else | |
| # Verify that the correct number of benes were imported into the database. | |
| CCLF_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CCLF_FILE" 2>&1` | |
| echo "CCLF_BENES: $CCLF_BENES" | |
| if [[ $? -ne 0 || -z $CCLF_BENES ]]; then | |
| echo "CCLF beneficiaries query returned zero results or command failed" | |
| exit 1 | |
| fi | |
| if [[ $(echo $CCLF_BENES | xargs) != "6" ]]; then | |
| echo "expected 6 beneficiaries imported from file, received $CCLF_BENES". | |
| exit 1 | |
| fi | |
| fi | |
| # Verify that we have a record of the CSV file in the database | |
| CSV_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CSVFILENAME' LIMIT 1" 2>&1` | |
| echo "CSV_FILE: $CSV_FILE" | |
| if [[ $? -ne 0 || -z $CSV_FILE ]]; then | |
| echo "csv_file query returned zero results or command failed" | |
| exit 1 | |
| else | |
| # Verify that the correct number of benes were imported into the database. | |
| CSV_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CSV_FILE" 2>&1` | |
| echo "CSV_BENES: $CSV_BENES" | |
| if [[ $? -ne 0 || -z $CSV_BENES ]]; then | |
| echo "CSV beneficiaries query returned zero results or command failed" | |
| exit 1 | |
| fi | |
| if [[ $(echo $CSV_BENES | xargs) != "5" ]]; then | |
| echo "expected 5 beneficiaries imported from file, received $CSV_BENES". | |
| exit 1 | |
| fi | |
| fi | |
| # Verify that we have a record of the GUIDE CSV file in the database | |
| CSV_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CSVFILENAME' LIMIT 1" 2>&1` | |
| echo "CSV_FILE: $CSV_FILE" | |
| if [[ $? -ne 0 || -z $CSV_FILE ]]; then | |
| echo "csv_file query returned zero results or command failed" | |
| exit 1 | |
| else | |
| # Verify that the correct number of benes were imported into the database. | |
| CSV_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CSV_FILE" 2>&1` | |
| echo "CSV_BENES: $CSV_BENES" | |
| if [[ $? -ne 0 || -z $CSV_BENES ]]; then | |
| echo "CSV beneficiaries query returned zero results or command failed" | |
| exit 1 | |
| fi | |
| if [[ $(echo $CSV_BENES | xargs) != "5" ]]; then | |
| echo "expected 5 beneficiaries imported from file, received $CSV_BENES". | |
| exit 1 | |
| fi | |
| fi | |