Adding Github_Token to workflows #3
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Data-Platform Terraform | ||
| env: | ||
| aws_deploy_region: "eu-west-2" | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| terraform_infra_import: | ||
| description: "Terraform import statements" | ||
| required: false | ||
| default: "" | ||
| type: string | ||
| terraform_infra_remove: | ||
| description: "Terraform state rm statements" | ||
| required: false | ||
| default: "" | ||
| type: string | ||
| environment: | ||
| description: "Environment to deploy to" | ||
| required: true | ||
| type: string | ||
| automation_build_url: | ||
| description: "Link to where the build pipeline is defined" | ||
| type: string | ||
| required: true | ||
| build_path: | ||
| required: true | ||
| type: string | ||
| terraform_state_s3_key_prefix: | ||
| required: true | ||
| type: string | ||
| terraform_state_file_name: | ||
| required: true | ||
| type: string | ||
| secrets: | ||
| GOOGLE_PROJECT_ID: | ||
| required: true | ||
| AWS_DEPLOY_ACCOUNT_ID: | ||
| description: "ID of Account being deployed to" | ||
| required: true | ||
| INFRASTRUCTURE_PRIVATE_KEY: | ||
| required: true | ||
| AWS_ACCESS_KEY_ID: | ||
| required: true | ||
| AWS_SECRET_ACCESS_KEY: | ||
| required: true | ||
| AWS_API_ACCOUNT_PROD: | ||
| required: true | ||
| AWS_MOSAIC_PROD_ACCOUNT_ID: | ||
| required: true | ||
| AWS_DATA_PLATFORM_ACCOUNT_ID: | ||
| required: true | ||
| AWS_HACKIT_ACCOUNT_ID: | ||
| required: true | ||
| AWS_SANDBOX_ACCOUNT_ID: | ||
| required: true | ||
| AWS_ROLE_TO_ASSUME: | ||
| required: true | ||
| AWS_API_VPC_ID: | ||
| required: true | ||
| AWS_HOUSING_VPC_ID: | ||
| required: true | ||
| AWS_MOSAIC_VPC_ID: | ||
| required: true | ||
| AWS_DP_VPC_ID: | ||
| required: true | ||
| GOOGLE_CREDENTIALS: | ||
| required: true | ||
| COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE: | ||
| required: true | ||
| PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN: | ||
| required: true | ||
| PRODUCTION_FIREWALL_IP: | ||
| required: true | ||
| TERRAFORM_SECRET_TOKEN: | ||
| required: true | ||
| GITHUB_TOKEN: | ||
| required: false | ||
| jobs: | ||
| deploy: | ||
| name: Terraform Apply | ||
| runs-on: ubuntu-24.04 | ||
| environment: ${{ inputs.environment }} | ||
| steps: | ||
| - name: Checkout Source | ||
| uses: actions/checkout@v3 | ||
| - name: Set Github Auth | ||
| run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com | ||
| shell: bash | ||
| - name: Install Terraform | ||
| uses: hashicorp/[email protected] | ||
| with: | ||
| terraform_wrapper: false | ||
| terraform_version: 1.2.0 | ||
| - name: Add a key to allow access to Infastructure | ||
| run: | | ||
| mkdir -p ~/.ssh | ||
| ssh-keyscan github.com >> ~/.ssh/known_hosts | ||
| echo "${{ secrets.INFRASTRUCTURE_PRIVATE_KEY }}" > ~/.ssh/id_rsa | ||
| chmod 400 ~/.ssh/id_rsa ~/.ssh/known_hosts | ||
| - name: Setup Node.js 14 | ||
| uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: "14" | ||
| - name: Install pipenv | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install pipenv | ||
| - name: Set up Google Cloud Credentials | ||
| run: | | ||
| echo $GOOGLE_CREDENTIALS >> ./google_service_account_creds.json | ||
| shell: bash | ||
| env: | ||
| GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }} | ||
| - name: Install Dependencies for rds-database-snapshot-replicator Lambda | ||
| working-directory: "./lambdas/rds-database-snapshot-replicator" | ||
| run: npm install | ||
| # - name: Install Dependencies for g-drive-to-s3 Lambda | ||
| # working-directory: "./lambdas/g_drive_to_s3" | ||
| # run: make install-requirements | ||
| - name: Download External Python Dependencies | ||
| working-directory: "./external-lib" | ||
| run: make all | ||
| - name: Package AWS Glue Job Helpers Module & Python modules | ||
| working-directory: "." | ||
| run: make package-helpers | ||
| - name: Setup deployment user profile | ||
| run: | | ||
| aws configure set default.region eu-west-2 > /dev/null 2>&1 | ||
| echo "[profile deploy_role]" >> ~/.aws/config | ||
| echo "role_arn=arn:aws:iam::${{ secrets.aws_deploy_account_id }}:role/${{ secrets.AWS_ROLE_TO_ASSUME }}" >> ~/.aws/config | ||
| echo "source_profile=default" >> ~/.aws/config | ||
| echo "role_session_name=deploy" >> ~/.aws/config | ||
| echo "region=eu-west-2" >> ~/.aws/config | ||
| - name: Deploy Data Platform in ${{ inputs.build_path }} | ||
| uses: ./.github/actions/terraform-deploy | ||
| with: | ||
| aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
| aws_deploy_region: ${{ env.aws_deploy_region }} | ||
| terraform_state_s3_key_prefix: ${{ inputs.terraform_state_s3_key_prefix }} | ||
| terraform_state_file_name: ${{ inputs.terraform_state_file_name }} | ||
| build_path: ${{ inputs.build_path }} | ||
| automation_build_url: ${{ inputs.automation_build_url }} | ||
| environment: ${{ inputs.environment }} | ||
| google_project_id: ${{ secrets.GOOGLE_PROJECT_ID }} | ||
| aws_deploy_account_id: ${{ secrets.AWS_DEPLOY_ACCOUNT_ID }} | ||
| aws_api_account_id: ${{ secrets.AWS_API_ACCOUNT_PROD }} | ||
| aws_mosaic_prod_account_id: ${{ secrets.AWS_MOSAIC_PROD_ACCOUNT_ID }} | ||
| aws_data_platform_account_id: ${{ secrets.AWS_DATA_PLATFORM_ACCOUNT_ID }} | ||
| aws_sandbox_account_id: ${{ secrets.AWS_SANDBOX_ACCOUNT_ID }} | ||
| aws_api_vpc_id: ${{ secrets.AWS_API_VPC_ID }} | ||
| aws_housing_vpc_id: ${{ secrets.AWS_HOUSING_VPC_ID }} | ||
| aws_mosaic_vpc_id: ${{ secrets.AWS_MOSAIC_VPC_ID }} | ||
| aws_dp_vpc_id: ${{ secrets.AWS_DP_VPC_ID }} | ||
| aws_hackit_account_id: ${{ secrets.AWS_HACKIT_ACCOUNT_ID }} | ||
| aws_deploy_iam_role_name: ${{ secrets.AWS_ROLE_TO_ASSUME }} | ||
| copy_liberator_to_pre_prod_lambda_execution_role: ${{ secrets.COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE }} | ||
| pre_production_liberator_data_storage_kms_key_arn: ${{ secrets.PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN }} | ||
| production_firewall_ip: ${{ secrets.PRODUCTION_FIREWALL_IP }} | ||
| branch: "main" | ||
| - name: Configure Redshift users, external schemas and user permissions for external schema | ||
| working-directory: "./terraform/etl" | ||
| if: ${{ (inputs.build_path == './terraform/etl') && (success()) }} | ||
| run: | | ||
| terraform_outputs=$(terraform output -json) | ||
| cd ../../ | ||
| source .venv/bin/activate | ||
| AWS_PROFILE=deploy_role python ./scripts/configure_redshift.py "$terraform_outputs" | ||
| - name: Build and Deploy SQL to Parquet Image to ECR | ||
| if: ${{ (inputs.build_path == './terraform/core') && (success()) }} | ||
| run: | | ||
| AWS_PROFILE=deploy_role ./docker/sql-to-parquet/deploy.sh | ||
| - name: Build and Deploy Pre-Production clean up job Image to ECR | ||
| if: ${{ (inputs.build_path == './terraform/core') && (success()) }} | ||
| run: | | ||
| ENVIRONMENT=${{ inputs.environment }} AWS_PROFILE=deploy_role ./docker/pre-production-data-cleanup/deploy.sh | ||
