feat(athena): FDOS-614 Add Athena stack with DynamoDB and RDS connectors #6750
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Application Deployment Pipeline | |
| permissions: | |
| id-token: write | |
| contents: write | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - "task/**" | |
| - "dependabot/**" | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'architecture/**' | |
| workflow_dispatch: | |
| # checkov:skip=CKV_GHA_7:Inputs reviewed and approved | |
| inputs: | |
| ref: | |
| description: "Specify the Git tag to be used for deployment (e.g. v1.1.0-pre.1)" | |
| required: false | |
| type: string | |
| environment: | |
| description: "Deployment environment" | |
| required: true | |
| default: "dev" | |
| type: choice | |
| options: | |
| - dev | |
| - test | |
| - sandpit | |
| - int | |
| - ref | |
| - prod | |
| jobs: | |
| metadata: | |
| name: "Get CI/CD metadata" | |
| uses: ./.github/workflows/metadata.yaml | |
| with: | |
| ref: ${{ inputs.ref }} | |
| quality-checks: | |
| name: "Quality checks for ${{ needs.metadata.outputs.environment }} deployment" | |
| needs: | |
| - metadata | |
| uses: ./.github/workflows/quality-checks.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui']" | |
| type: app | |
| build_timestamp: ${{ needs.metadata.outputs.build_timestamp }} | |
| skip_dirs: "services/dos-ui,services/read-only-viewer" | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| MGMT_ACCOUNT_ID: ${{ secrets.MGMT_ACCOUNT_ID }} | |
| IDP_AWS_REPORT_UPLOAD_ACCOUNT_ID: ${{ secrets.IDP_AWS_REPORT_UPLOAD_ACCOUNT_ID }} | |
| IDP_AWS_REPORT_UPLOAD_REGION: ${{ secrets.IDP_AWS_REPORT_UPLOAD_REGION }} | |
| IDP_AWS_REPORT_UPLOAD_ROLE_NAME: ${{ secrets.IDP_AWS_REPORT_UPLOAD_ROLE_NAME }} | |
| IDP_AWS_REPORT_UPLOAD_BUCKET_ENDPOINT: ${{ secrets.IDP_AWS_REPORT_UPLOAD_BUCKET_ENDPOINT }} | |
| prepare-toggle-artifacts: | |
| name: "Prepare Toggle artifacts" | |
| needs: | |
| - metadata | |
| - quality-checks | |
| uses: ./.github/workflows/prepare-toggle-artifacts.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| ref: ${{ inputs.ref }} | |
| check-stack-toggles: | |
| name: "Check stack toggles for ${{ needs.metadata.outputs.environment }}" | |
| needs: | |
| - metadata | |
| - prepare-toggle-artifacts | |
| uses: ./.github/workflows/check-stack-toggles.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| ref: ${{ inputs.ref }} | |
| build-services: | |
| name: "Build ${{ matrix.name }}" | |
| needs: | |
| - metadata | |
| - quality-checks | |
| - check-stack-toggles | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: "python" | |
| build_type: "package" | |
| build_enabled: true | |
| - name: "crud-apis" | |
| build_type: "service" | |
| build_enabled: true | |
| - name: "data-migration" | |
| build_type: "service" | |
| build_enabled: true | |
| - name: "read-only-viewer" | |
| build_type: "service" | |
| build_enabled: ${{ needs.check-stack-toggles.outputs.read_only_viewer_enabled == 'true' }} | |
| - name: "etl-ods" | |
| build_type: "service" | |
| build_enabled: true | |
| - name: "dos-search" | |
| build_type: "service" | |
| build_enabled: true | |
| - name: "dos-ui" | |
| build_type: "service" | |
| build_enabled: ${{ needs.check-stack-toggles.outputs.ui_enabled == 'true' }} | |
| uses: ./.github/workflows/build-project.yaml | |
| with: | |
| name: ${{ matrix.name }} | |
| build_type: ${{ matrix.build_type }} | |
| python_version: ${{ needs.metadata.outputs.python_version }} | |
| commit_hash: ${{ needs.metadata.outputs.commit_hash }} | |
| environment: ${{ needs.metadata.outputs.mgmt_environment }} | |
| repo_name: ${{ needs.metadata.outputs.reponame }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| type: app | |
| ref: ${{ inputs.ref }} | |
| build_enabled: ${{ matrix.build_enabled }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| build-sandbox-containers: | |
| name: "Build container ${{ matrix.name }}" | |
| needs: | |
| - metadata | |
| - quality-checks | |
| strategy: | |
| matrix: | |
| include: | |
| - name: "sandbox-dos-search" | |
| uses: ./.github/workflows/build-sandbox-images.yaml | |
| with: | |
| name: ${{ matrix.name }} | |
| commit_hash: ${{ needs.metadata.outputs.commit_hash }} | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| PROXYGEN_URL: ${{ secrets.PROXYGEN_URL }} | |
| perform-code-analysis: | |
| name: "Perform static code analysis" | |
| needs: | |
| - metadata | |
| - build-services | |
| uses: ./.github/workflows/static-code-analysis.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| deploy-toggle-infrastructure: | |
| name: "Deploy toggle infrastructure to the ${{ needs.metadata.outputs.environment }} environment" | |
| needs: | |
| - metadata | |
| - build-services | |
| - check-stack-toggles | |
| uses: ./.github/workflows/deploy-application-infrastructure.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| ref: ${{ inputs.ref }} | |
| workflow_timeout: 30 | |
| stacks: "['app_config']" | |
| skip_manual_approval: "true" | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| MGMT_ACCOUNT_ID: ${{ secrets.MGMT_ACCOUNT_ID }} | |
| deploy-application-infrastructure: | |
| name: "Deploy application infrastructure to the ${{ needs.metadata.outputs.environment }} environment" | |
| needs: | |
| - metadata | |
| - build-services | |
| - check-stack-toggles | |
| - deploy-toggle-infrastructure | |
| uses: ./.github/workflows/deploy-application-infrastructure.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| ref: ${{ inputs.ref }} | |
| workflow_timeout: 30 | |
| stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui']" | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| MGMT_ACCOUNT_ID: ${{ secrets.MGMT_ACCOUNT_ID }} | |
| deploy-proxy-to-apim: | |
| name: "Deploy ${{ matrix.api_name }} API to Proxygen" | |
| needs: | |
| - metadata | |
| - deploy-application-infrastructure | |
| uses: ./.github/workflows/authenticate-and-deploy-to-apim.yaml | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| api_name: | |
| - dos-search | |
| - dos-ingest | |
| # Add more API names here as needed | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| api_name: ${{ matrix.api_name }} | |
| apim_env: ${{ needs.metadata.outputs.environment == 'dev' && 'internal-dev' || needs.metadata.outputs.environment == 'test' && 'internal-qa' || needs.metadata.outputs.environment }} | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| PROXYGEN_URL: ${{ secrets.PROXYGEN_URL }} | |
| # TODO : FTRS-1899 Re-enable data migration step once ETL process is stable | |
| # migrate-data: | |
| # name: "Run ETL process in ${{ needs.metadata.outputs.environment }}" | |
| # if: github.ref == 'refs/heads/main' && needs.metadata.outputs.environment != 'prod' | |
| # concurrency: | |
| # group: "${{ needs.metadata.outputs.environment }}-${{ needs.metadata.outputs.workspace }}" | |
| # cancel-in-progress: false | |
| # needs: | |
| # - metadata | |
| # - deploy-application-infrastructure | |
| # uses: ./.github/workflows/migrate-data.yaml | |
| # with: | |
| # environment: ${{ needs.metadata.outputs.environment }} | |
| # function_name: "ftrs-dos-${{ needs.metadata.outputs.environment }}-data-migration-queue-populator-lambda" | |
| # queue_name: "ftrs-dos-${{ needs.metadata.outputs.environment }}-data-migration-dms-events" | |
| # type: app | |
| # ref: ${{ inputs.ref }} | |
| # secrets: inherit | |
| export-dynamodb-to-s3: | |
| name: "Export DynamoDB tables to S3 in ${{ needs.metadata.outputs.environment }}" | |
| if: github.ref == 'refs/heads/main' && needs.metadata.outputs.environment != 'prod' | |
| concurrency: | |
| group: "${{ needs.metadata.outputs.environment }}-${{ needs.metadata.outputs.workspace }}" | |
| cancel-in-progress: false | |
| needs: | |
| - metadata | |
| - deploy-application-infrastructure | |
| # - migrate-data | |
| uses: ./.github/workflows/manage-dynamodb-data.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| type: app | |
| action: export | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| restore-dynamodb-from-s3: | |
| name: "Restore data from S3 to DynamoDB tables in ${{ needs.metadata.outputs.workspace }}" | |
| if: needs.metadata.outputs.workspace != 'default' && needs.metadata.outputs.environment != 'prod' | |
| needs: | |
| - metadata | |
| - deploy-application-infrastructure | |
| uses: ./.github/workflows/manage-dynamodb-data.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| action: import | |
| type: app | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| deploy-open-search-indexes: | |
| name: "Deploy OpenSearch indexes to ${{ needs.metadata.outputs.environment }}" | |
| if: false && needs.check-stack-toggles.outputs.open_search_enabled == 'true' | |
| needs: | |
| - metadata | |
| - check-stack-toggles | |
| - deploy-application-infrastructure | |
| - restore-dynamodb-from-s3 | |
| uses: ./.github/workflows/deploy-open-search-indexes.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| aws_region: ${{ vars.AWS_REGION }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| deploy-frontend-services: | |
| name: "Deploy ${{ matrix.name }} to ${{ needs.metadata.outputs.environment }}" | |
| concurrency: | |
| group: "${{ needs.metadata.outputs.environment }}-${{ needs.metadata.outputs.workspace }}-${{ matrix.name }}" | |
| cancel-in-progress: false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: "dos-ui" | |
| enabled: ${{ needs.check-stack-toggles.outputs.ui_enabled }} | |
| - name: "read-only-viewer" | |
| enabled: ${{ needs.check-stack-toggles.outputs.read_only_viewer_enabled }} | |
| needs: | |
| - metadata | |
| - check-stack-toggles | |
| - deploy-application-infrastructure | |
| uses: ./.github/workflows/deploy-frontend-project.yaml | |
| with: | |
| name: ${{ matrix.name }} | |
| enabled: ${{ matrix.enabled }} | |
| build_type: "service" | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| repo_name: ${{ needs.metadata.outputs.reponame }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| type: "app" | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| deploy-data-migration-service: | |
| name: "Deploy data migration service to ${{ needs.metadata.outputs.environment }}" | |
| if: github.ref == 'refs/heads/main' && needs.metadata.outputs.environment != 'prod' | |
| concurrency: | |
| group: "${{ needs.metadata.outputs.environment }}-data-migration-${{ needs.metadata.outputs.workspace }}" | |
| cancel-in-progress: false | |
| needs: | |
| - metadata | |
| - deploy-application-infrastructure | |
| uses: ./.github/workflows/deploy-data-migration-project.yaml | |
| with: | |
| name: "data-migration" | |
| build_type: "service" | |
| python_version: ${{ needs.metadata.outputs.python_version }} | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| repo_name: ${{ needs.metadata.outputs.reponame }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| type: "app" | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| service-automation-tests: | |
| name: "Run ${{ matrix.tag }} service automation tests on ${{ needs.metadata.outputs.environment }}" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| include: | |
| - tag: "integrated-search" | |
| type: "bdd" | |
| api_name: "dos-search" | |
| - tag: "data-migration" | |
| type: "bdd" | |
| - tag: "data-sourcing" | |
| type: "bdd" | |
| needs: | |
| - metadata | |
| - deploy-application-infrastructure | |
| - restore-dynamodb-from-s3 | |
| - export-dynamodb-to-s3 | |
| - deploy-proxy-to-apim | |
| if: | | |
| always() && | |
| !cancelled() && | |
| ( | |
| needs.restore-dynamodb-from-s3.result == 'success' || | |
| ( | |
| needs.restore-dynamodb-from-s3.result == 'skipped' && | |
| (needs.export-dynamodb-to-s3.result == 'success' || needs.deploy-application-infrastructure.result == 'success') | |
| ) | |
| ) | |
| uses: ./.github/workflows/service-automation-test.yaml | |
| with: | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| workspace: ${{ needs.metadata.outputs.workspace }} | |
| commit_hash: ${{ needs.metadata.outputs.commit_hash }} | |
| ref: ${{ inputs.ref }} | |
| test_tag: ${{ matrix.tag }} | |
| test_type: ${{ matrix.type }} | |
| api_name: ${{ matrix.api_name || '' }} | |
| type: app | |
| deployment_type: "development" | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| MGMT_ACCOUNT_ID: ${{ secrets.MGMT_ACCOUNT_ID }} | |
| generate-prerelease: | |
| name: "Generate prerelease tag" | |
| needs: | |
| - deploy-frontend-services | |
| - service-automation-tests | |
| if: > | |
| always() && | |
| github.event_name == 'push' && | |
| github.ref == 'refs/heads/main' && | |
| (needs.deploy-frontend-services.result == 'success' || needs.deploy-frontend-services.result == 'skipped') && | |
| needs.service-automation-tests.result == 'success' | |
| uses: ./.github/workflows/generate-prerelease.yaml | |
| secrets: inherit | |
| stage-prerelease-artefacts: | |
| name: "Stage ${{ matrix.name }} artefacts for prerelease ${{ needs.generate-prerelease.outputs.prerelease_tag }}" | |
| needs: | |
| - metadata | |
| - generate-prerelease | |
| uses: ./.github/workflows/promote-artefacts.yaml | |
| with: | |
| repo_name: ${{ needs.metadata.outputs.reponame }} | |
| artefact_promotion_type: "stage" | |
| type: app | |
| prerelease_tag: ${{ needs.generate-prerelease.outputs.prerelease_tag }} | |
| ref: ${{ inputs.ref }} | |
| secrets: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| check-pipeline-status: | |
| name: "Check Pipeline Status" | |
| needs: | |
| - deploy-application-infrastructure | |
| - deploy-frontend-services | |
| - deploy-data-migration-service | |
| - service-automation-tests | |
| - export-dynamodb-to-s3 | |
| - stage-prerelease-artefacts | |
| if: always() | |
| uses: ./.github/workflows/pipeline-status-check.yaml | |
| slack-notifications: | |
| name: "Send Notification to Slack" | |
| needs: | |
| - metadata | |
| - quality-checks | |
| - build-services | |
| - deploy-application-infrastructure | |
| # - migrate-data | |
| - export-dynamodb-to-s3 | |
| - restore-dynamodb-from-s3 | |
| - deploy-frontend-services | |
| - deploy-data-migration-service | |
| - service-automation-tests | |
| - generate-prerelease | |
| - stage-prerelease-artefacts | |
| - check-pipeline-status | |
| if: always() | |
| uses: ./.github/workflows/slack-notifications.yaml | |
| with: | |
| env: ${{ needs.metadata.outputs.environment }} | |
| secrets: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} |