cclf-import-integration-test.yml

# Integration test that triggers a CCLF import in the test environment and
# verifies the imported data in the test database for ACO A0001.
name: cclf-import test integration

on:
  pull_request:
    paths:
      - .github/workflows/cclf-import-integration-test.yml
      - .github/workflows/cclf-import-deploy.yml
      - bcda/cclf/**
      - bcda/lambda/cclf/**
  workflow_dispatch:

permissions:
  contents: read
  id-token: write

# Ensure we have only one integration test running at a time
concurrency:
  group: cclf-import-test-integration

jobs:
  # Deploy first if triggered by pull_request
  deploy:
    if: ${{ github.event_name == 'pull_request' }}
    uses: ./.github/workflows/cclf-import-deploy.yml
    secrets: inherit
    with:
      deploy_env: test

  trigger:
    if: ${{ always() }}
    needs: deploy
    runs-on: codebuild-bcda-app-${{github.run_id}}-${{github.run_attempt}}
    defaults:
      run:
        working-directory: bcda
    steps:
      - uses: actions/checkout@v4
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: ${{ vars.AWS_REGION }}
          role-to-assume: arn:aws:iam::${{ secrets.NON_PROD_ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-cclf-import-function
      - name: Get BFD Account
        uses: cmsgov/cdap/actions/aws-params-env-action@main
        env:
          AWS_REGION: ${{ vars.AWS_REGION }}
        with:
          params: | 
            BFD_ACCOUNT_ID=/bfd/account-id
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: ${{ vars.AWS_REGION }}
          # Note that we use the BFD role with access to the bucket
          role-to-assume: arn:aws:iam::${{ env.BFD_ACCOUNT_ID }}:role/delegatedadmin/developer/bfd-test-eft-bcda-ct-bucket-role
          role-chaining: true
          role-skip-session-tagging: true
      - name: Upload test file to the BFD bucket to trigger lambda function via SNS message
        run: |
          year=$(date +'%y')
          date=$(date +'%y%m%d')
          time=$(date +'%H%M%S')

          fname=T.BCD.A0001.ZCY${year}.D${date}.T${time}1
          cclf8_fname=T.BCD.A0001.ZC8Y${year}.D${date}.T${time}1
          echo "CCLFFILENAME=$cclf8_fname" >> "$GITHUB_ENV"

          csvname=T.PCPB.M${year}11.D${date}.T${time}1
          echo "CSVFILENAME=$csvname" >> "$GITHUB_ENV"

          guidecsvname=T.GUIDE.GUIDE-00001.Y${year}.D${date}.T${time}1
          echo "GUIDEFILENAME=$guidecsvname" >> "$GITHUB_ENV"

          mv ../shared_files/cclf/archives/csv/P.PCPB.M2411.D181120.T1000000 ${csvname}

          mv ../shared_files/cclf/archives/csv/T.GUIDE.GUIDE-00001.Y25.D250630.T1000000 ${guidecsvname}

          unzip ../shared_files/cclf/archives/valid/T.BCD.A0001.ZCY18.D181120.T1000000

          # Clone the synthetic zip file with updated years
          new_dir="D${date}.T${time}"
          mkdir $new_dir
          mv T.BCD.A0001.ZC8Y18.D181120.T1000009 $new_dir/T.BCD.A0001.ZC8Y${year}.D${date}.T${time}1
          mv T.BCD.A0001.ZC9Y18.D181120.T1000010 $new_dir/T.BCD.A0001.ZC9Y${year}.D${date}.T${time}1
          mv T.BCD.A0001.ZC0Y18.D181120.T1000011 $new_dir/T.BCD.A0001.ZC0Y${year}.D${date}.T${time}1
          zip -jr $fname $new_dir

          aws s3 cp --no-progress $fname \
            s3://bfd-test-eft/bfdeft01/bcda/in/test/$fname

          aws s3 cp --no-progress ${csvname} \
            s3://bfd-test-eft/bfdeft01/bcda/in/test/${csvname}

          aws s3 cp --no-progress ${guidecsvname} \
            s3://bfd-test-eft/bfdeft01/bcda/in/test/${guidecsvname}
      # Give a bit of time for lambdas to process files
      - run: sleep 60
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: ${{ vars.AWS_REGION }}
          role-to-assume: arn:aws:iam::${{ secrets.NON_PROD_ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-github-actions
      - name: Install psql
        run: |
          sudo dnf -y update
          sudo dnf install postgresql16 -y
      - name: Get database credentials
        uses: cmsgov/cdap/actions/aws-params-env-action@main
        env:
          AWS_REGION: ${{ vars.AWS_REGION }}
        with:
          params: |
            CONNECTION_INFO=/bcda/test/api/DATABASE_URL
      - name: Verify CCLF file was ingested
        env:
          PGSSLMODE: require
        # CAUTION: if changing the script below, validate that sensitive information is not printed in the workflow
        run: |
          HOST=$(aws rds describe-db-cluster-endpoints --db-cluster-identifier bcda-test-aurora 2>&1 | jq -r '.DBClusterEndpoints[0].Endpoint' 2>&1)
          CONNECTION_URL=$(echo $CONNECTION_INFO 2>&1 | sed -E "s/@.*\/bcda/\@$HOST\/bcda/" 2>&1)

          # Verify that we have a record of the CCLF file in the database
          CCLF_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CCLFFILENAME' LIMIT 1" 2>&1`
          echo "CCLF_FILE: $CCLF_FILE"
          if [[ $? -ne 0 || -z $CCLF_FILE ]]; then
            echo "cclf_file query returned zero results or command failed"
            exit 1
          else
            
            # Verify that the correct number of benes were imported into the database.
            CCLF_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CCLF_FILE" 2>&1`
            echo "CCLF_BENES: $CCLF_BENES"
            if [[ $? -ne 0 || -z $CCLF_BENES ]]; then
              echo "CCLF beneficiaries query returned zero results or command failed"
              exit 1
            fi
            if [[ $(echo $CCLF_BENES | xargs) != "6" ]]; then
              echo "expected 6 beneficiaries imported from file, received $CCLF_BENES".
              exit 1
            fi
          fi

          # Verify that we have a record of the CSV file in the database
          CSV_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CSVFILENAME' LIMIT 1" 2>&1`
          echo "CSV_FILE: $CSV_FILE"
          if [[ $? -ne 0 || -z $CSV_FILE ]]; then
            echo "csv_file query returned zero results or command failed"
            exit 1
          else
            
            # Verify that the correct number of benes were imported into the database.
            CSV_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CSV_FILE" 2>&1`
            echo "CSV_BENES: $CSV_BENES"
            if [[ $? -ne 0 || -z $CSV_BENES ]]; then
              echo "CSV beneficiaries query returned zero results or command failed"
              exit 1
            fi
            if [[ $(echo $CSV_BENES | xargs) != "5" ]]; then
              echo "expected 5 beneficiaries imported from file, received $CSV_BENES".
              exit 1
            fi
          fi

          # Verify that we have a record of the GUIDE CSV file in the database
          CSV_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CSVFILENAME' LIMIT 1" 2>&1`
          echo "CSV_FILE: $CSV_FILE"
          if [[ $? -ne 0 || -z $CSV_FILE ]]; then
            echo "csv_file query returned zero results or command failed"
            exit 1
          else
            
            # Verify that the correct number of benes were imported into the database.
            CSV_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CSV_FILE" 2>&1`
            echo "CSV_BENES: $CSV_BENES"
            if [[ $? -ne 0 || -z $CSV_BENES ]]; then
              echo "CSV beneficiaries query returned zero results or command failed"
              exit 1
            fi
            if [[ $(echo $CSV_BENES | xargs) != "5" ]]; then
              echo "expected 5 beneficiaries imported from file, received $CSV_BENES".
              exit 1
            fi
          fi