diff --git a/.github/aws/web.yml b/.github/aws/web.yml new file mode 100644 index 0000000..0b1d3a1 --- /dev/null +++ b/.github/aws/web.yml @@ -0,0 +1,151 @@ +family: "$ECS_WEB_TASK" +networkMode: awsvpc +cpu: "$ECS_WEB_TASK_CPU_UNITS" +memory: "$ECS_WEB_TASK_MEMORY_UNITS" +executionRoleArn: "$ROLE_ARN" +taskRoleArn: "$ROLE_ARN" +requiresCompatibilities: + - FARGATE +volumes: + - name: data + efsVolumeConfiguration: + fileSystemId: "$EFS_FILESYSTEM_ID" + authorizationConfig: + accessPointId: "$EFS_ACCESS_POINT_ID" + iam: ENABLED + transitEncryption: ENABLED +containerDefinitions: + - name: logs + image: public.ecr.aws/aws-observability/aws-for-fluent-bit:stable + firelensConfiguration: + type: fluentbit + memoryReservation: 50 + logConfiguration: + logDriver: awslogs + options: + awslogs-group: "/analysistools/$TIER/$APP/web" + awslogs-region: "$AWS_REGION" + awslogs-stream-prefix: logs + + - name: frontend + image: "$FRONTEND_IMAGE_LATEST" + portMappings: + - protocol: tcp + containerPort: $FRONTEND_CONTAINER_PORT + environment: + - name: API_BASE_URL + value: http://localhost:$BACKEND_CONTAINER_PORT + secrets: + - name: APP_PATH + valueFrom: "/analysistools/$TIER/$APP/application_path" + - name: SERVER_TIMEOUT + valueFrom: "/analysistools/$TIER/$APP/server_timeout" + logConfiguration: + logDriver: awsfirelens + options: + Name: datadog + tls: "on" + tls.verify: "off" + dd_service: "$TIER-$APP-frontend" + dd_source: "httpd" + dd_tags: "project:$APP tier:$TIER" + provider: ecs + secretOptions: + - name: Host + valueFrom: /analysistools/$TIER/datadog/log_endpoint_host + - name: apikey + valueFrom: /analysistools/$TIER/datadog/api_key + memoryReservation: 100 + + - name: backend + image: "$BACKEND_IMAGE_LATEST" + environment: + - name: AWS_DEFAULT_REGION + value: "$AWS_REGION" + - name: APP_NAME + value: "$APP" + - name: APP_PORT + value: "$BACKEND_CONTAINER_PORT" + - name: APP_TIER + value: "$TIER" + - name: WORKER_TYPE + value: "fargate" + - name: "TZ" + value: "$TZ" + secrets: + - name: APP_BASE_URL + valueFrom: "/analysistools/$TIER/$APP/base_url" + - name: API_BASE_URL + valueFrom: "/analysistools/$TIER/$APP/base_url" + - name: LOG_LEVEL + valueFrom: "/analysistools/$TIER/datadog/log_level" + - name: SERVER_TIMEOUT + valueFrom: "/analysistools/$TIER/$APP/server_timeout" + - name: APP_DATA_FOLDER + valueFrom: "/analysistools/$TIER/$APP/app_data_folder" + - name: APP_SCRIPTS + valueFrom: "/analysistools/$TIER/$APP/app_scripts" + - name: DATA_FOLDER + valueFrom: "/analysistools/$TIER/$APP/data_folder" + - name: DATA_BUCKET + valueFrom: "/analysistools/$TIER/$APP/data_bucket" + - name: DATA_BUCKET_PREFIX + valueFrom: "/analysistools/$TIER/$APP/data_bucket_prefix" + - name: INPUT_FOLDER + valueFrom: "/analysistools/$TIER/$APP/input_folder" + - name: INPUT_KEY_PREFIX + valueFrom: "/analysistools/$TIER/$APP/input_key_prefix" + - name: IO_BUCKET + valueFrom: "/analysistools/$TIER/$APP/io_bucket" + - name: OUTPUT_FOLDER + valueFrom: "/analysistools/$TIER/$APP/output_folder" + - name: OUTPUT_KEY_PREFIX + valueFrom: "/analysistools/$TIER/$APP/output_key_prefix" + - name: VPC_ID + valueFrom: "/analysistools/$TIER/$APP/vpc_id" + - name: SUBNET_IDS + valueFrom: "/analysistools/$TIER/$APP/subnet_ids" + - name: SECURITY_GROUP_IDS + valueFrom: "/analysistools/$TIER/$APP/security_group_ids" + - name: ECS_CLUSTER + valueFrom: "/analysistools/$TIER/$APP/ecs_cluster" + - name: WORKER_TASK_NAME + valueFrom: "/analysistools/$TIER/$APP/ecs_worker_task" + - name: EMAIL_ADMIN + valueFrom: "/analysistools/$TIER/$APP/email_admin" + - name: EMAIL_SMTP_HOST + valueFrom: "/analysistools/$TIER/$APP/email_smtp_host" + - name: EMAIL_SMTP_PORT + valueFrom: "/analysistools/$TIER/$APP/email_smtp_port" + - name: EMAIL_TECH_SUPPORT + valueFrom: "/analysistools/$TIER/$APP/email_tech_support" + mountPoints: + - sourceVolume: data + containerPath: "/data" + readOnly: false + logConfiguration: + logDriver: awsfirelens + options: + Name: datadog + tls: "on" + tls.verify: "off" + dd_service: "$TIER-$APP-backend" + dd_source: "nodejs" + dd_tags: "project:$APP tier:$TIER" + provider: ecs + secretOptions: + - name: Host + valueFrom: /analysistools/$TIER/datadog/log_endpoint_host + - name: apikey + valueFrom: /analysistools/$TIER/datadog/api_key +tags: + - key: Project + value: "$APP" + - key: ResourceName + value: "$TIER-$APP-web-ecs-task" + - key: EnvironmentTier + value: "$ENVIRONMENT_TIER" + - key: ResourceFunction + value: compute + - key: Creator + value: TF diff --git a/.github/aws/worker.yml b/.github/aws/worker.yml new file mode 100644 index 0000000..2ba6700 --- /dev/null +++ b/.github/aws/worker.yml @@ -0,0 +1,121 @@ +family: "$ECS_WORKER_TASK" +networkMode: awsvpc +cpu: "$ECS_WORKER_TASK_CPU_UNITS" +memory: "$ECS_WORKER_TASK_MEMORY_UNITS" +executionRoleArn: "$ROLE_ARN" +taskRoleArn: "$ROLE_ARN" +requiresCompatibilities: + - FARGATE +volumes: + - name: data + efsVolumeConfiguration: + fileSystemId: "$EFS_FILESYSTEM_ID" + authorizationConfig: + accessPointId: "$EFS_ACCESS_POINT_ID" + iam: ENABLED + transitEncryption: ENABLED +containerDefinitions: + - name: logs + image: public.ecr.aws/aws-observability/aws-for-fluent-bit:stable + firelensConfiguration: + type: fluentbit + memoryReservation: 50 + logConfiguration: + logDriver: awslogs + options: + awslogs-group: "/analysistools/$TIER/$APP/worker" + awslogs-region: "$AWS_REGION" + awslogs-stream-prefix: logs + + - name: "worker" + image: "$BACKEND_IMAGE_LATEST" + environment: + - name: AWS_DEFAULT_REGION + value: "$AWS_REGION" + - name: APP_NAME + value: "$APP" + - name: APP_PORT + value: "$BACKEND_CONTAINER_PORT" + - name: APP_TIER + value: "$TIER" + - name: WORKER_TYPE + value: "fargate" + - name: "TZ" + value: "$TZ" + secrets: + - name: APP_BASE_URL + valueFrom: "/analysistools/$TIER/$APP/base_url" + - name: API_BASE_URL + valueFrom: "/analysistools/$TIER/$APP/base_url" + - name: LOG_LEVEL + valueFrom: "/analysistools/$TIER/datadog/log_level" + - name: SERVER_TIMEOUT + valueFrom: "/analysistools/$TIER/$APP/server_timeout" + - name: APP_DATA_FOLDER + valueFrom: "/analysistools/$TIER/$APP/app_data_folder" + - name: APP_SCRIPTS + valueFrom: "/analysistools/$TIER/$APP/app_scripts" + - name: DATA_FOLDER + valueFrom: "/analysistools/$TIER/$APP/data_folder" + - name: DATA_BUCKET + valueFrom: "/analysistools/$TIER/$APP/data_bucket" + - name: DATA_BUCKET_PREFIX + valueFrom: "/analysistools/$TIER/$APP/data_bucket_prefix" + - name: INPUT_FOLDER + valueFrom: "/analysistools/$TIER/$APP/input_folder" + - name: INPUT_KEY_PREFIX + valueFrom: "/analysistools/$TIER/$APP/input_key_prefix" + - name: IO_BUCKET + valueFrom: "/analysistools/$TIER/$APP/io_bucket" + - name: OUTPUT_FOLDER + valueFrom: "/analysistools/$TIER/$APP/output_folder" + - name: OUTPUT_KEY_PREFIX + valueFrom: "/analysistools/$TIER/$APP/output_key_prefix" + - name: VPC_ID + valueFrom: "/analysistools/$TIER/$APP/vpc_id" + - name: SUBNET_IDS + valueFrom: "/analysistools/$TIER/$APP/subnet_ids" + - name: SECURITY_GROUP_IDS + valueFrom: "/analysistools/$TIER/$APP/security_group_ids" + - name: ECS_CLUSTER + valueFrom: "/analysistools/$TIER/$APP/ecs_cluster" + - name: WORKER_TASK_NAME + valueFrom: "/analysistools/$TIER/$APP/ecs_worker_task" + - name: EMAIL_ADMIN + valueFrom: "/analysistools/$TIER/$APP/email_admin" + - name: EMAIL_SMTP_HOST + valueFrom: "/analysistools/$TIER/$APP/email_smtp_host" + - name: EMAIL_SMTP_PORT + valueFrom: "/analysistools/$TIER/$APP/email_smtp_port" + - name: EMAIL_TECH_SUPPORT + valueFrom: "/analysistools/$TIER/$APP/email_tech_support" + mountPoints: + - sourceVolume: data + containerPath: "/data" + readOnly: false + logConfiguration: + logDriver: awsfirelens + options: + Name: datadog + tls: "on" + tls.verify: "off" + dd_service: "$TIER-$APP-worker" + dd_source: "nodejs" + dd_tags: "project:$APP tier:$TIER" + provider: ecs + secretOptions: + - name: Host + valueFrom: /analysistools/$TIER/datadog/log_endpoint_host + - name: apikey + valueFrom: /analysistools/$TIER/datadog/api_key +tags: + - key: Project + value: "$APP" + - key: ResourceName + value: "$TIER-$APP-worker-ecs-task" + - key: EnvironmentTier + value: "$ENVIRONMENT_TIER" + - key: ResourceFunction + value: compute + - key: Creator + value: TF diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..f159241 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,227 @@ +name: Deploy ezQTL Application +on: + workflow_dispatch: + inputs: + tier: + description: "Tier to deploy to" + required: true + default: "dev" + type: choice + options: + - dev + - qa + - stage + - prod + +permissions: + id-token: write +jobs: + Deploy: + permissions: + contents: "read" + id-token: "write" + runs-on: ubuntu-latest + environment: ${{ inputs.tier || (endsWith(github.ref_name, '_dev') && 'dev' || 'qa') }} + env: + APP: ezqtl + TZ: America/New_York + AWS_REGION: us-east-1 + TASK_DEFINITION_TEMPLATE_PATH: aws + DOCKER_BUILDKIT: 1 + FRONTEND_CONTAINER_PORT: 80 + BACKEND_CONTAINER_PORT: 9000 + ECS_WEB_CPU_UNITS: "2 vCPU" + ECS_WEB_MEMORY_UNITS: "4 GB" + ECS_WORKER_CPU_UNITS: "4 vCPU" + ECS_WORKER_MEMORY_UNITS: "16 GB" + CICD_ROLE_ARN: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-cicd + + + steps: + - uses: "actions/checkout@v5" + + - name: Set environment variables + run: | + # Determine deployment tier + BRANCH="${{ github.ref_name }}" + [[ "${{ github.event_name }}" == "workflow_dispatch" ]] \ + && TIER="${{ inputs.tier }}" \ + || TIER="${BRANCH##*_}" + + # Set tier-dependent variables + [[ "$TIER" =~ ^(dev|qa)$ ]] && IMAGE_TIER="development" || IMAGE_TIER="release" + [[ "$TIER" == "dev" ]] && LOG_LEVEL="debug" || LOG_LEVEL="info" + + # Parse version info from branch name + IFS='_' read -r _ VERSION DATE <<< "$BRANCH" + + # Build image tags + TIMESTAMP=$(date +"%Y%m%d%H%M%S") + REPO="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$APP" + + # Export environment variables + cat << EOF >> $GITHUB_ENV + TIER=$TIER + IMAGE_TIER=$IMAGE_TIER + LOG_LEVEL=$LOG_LEVEL + VERSION=${VERSION:-unknown_version} + DATE=${DATE:-unknown_date} + IMAGE_REPOSITORY=$REPO + FRONTEND_IMAGE=$REPO:$IMAGE_TIER-frontend-$BRANCH-$TIMESTAMP + BACKEND_IMAGE=$REPO:$IMAGE_TIER-backend-$BRANCH-$TIMESTAMP + FRONTEND_IMAGE_LATEST=$REPO:$IMAGE_TIER-frontend-$BRANCH-latest + BACKEND_IMAGE_LATEST=$REPO:$IMAGE_TIER-backend-$BRANCH-latest + PARAMETER_PATH=/analysistools/$TIER/$APP + ENVIRONMENT_TIER=${TIER^^} + EOF + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5.1.0 + with: + role-to-assume: ${{ env.CICD_ROLE_ARN }} + role-session-name: ${{ env.TIER }}-${{ env.APP }}-deploy-${{ env.BRANCH }} + aws-region: ${{ env.AWS_REGION }} + + - name: Get parameters from AWS SSM + run: | + # Define parameters to retrieve (SSM name = ENV_VAR_NAME) + declare -A PARAMS=( + ["ecs_cluster"]="ECS_CLUSTER" + ["ecs_web_task"]="ECS_WEB_TASK" + ["ecs_web_task_cpu_units"]="ECS_WEB_TASK_CPU_UNITS" + ["ecs_web_task_memory_units"]="ECS_WEB_TASK_MEMORY_UNITS" + ["ecs_web_service"]="ECS_WEB_SERVICE" + ["ecs_worker_task"]="ECS_WORKER_TASK" + ["ecs_worker_task_cpu_units"]="ECS_WORKER_TASK_CPU_UNITS" + ["ecs_worker_task_memory_units"]="ECS_WORKER_TASK_MEMORY_UNITS" + ["role_arn"]="ROLE_ARN" + ["efs_filesystem_id"]="EFS_FILESYSTEM_ID" + ["efs_access_point_id"]="EFS_ACCESS_POINT_ID" + ) + + # Retrieve each parameter and export to environment + for ssm_name in "${!PARAMS[@]}"; do + env_var="${PARAMS[$ssm_name]}" + value=$(aws ssm get-parameter \ + --name "${{ env.PARAMETER_PATH }}/${ssm_name}" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text) + echo "${env_var}=${value}" >> $GITHUB_ENV + echo "Retrieved ${ssm_name} -> ${env_var}" + done + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create and use a new builder instance + run: | + docker buildx create --name mybuilder --use + + - name: Build backend image ${{ env.BACKEND_IMAGE }} + uses: docker/build-push-action@v6 + with: + context: . + file: docker/backend.dockerfile + pull: true + push: true + tags: | + ${{ env.BACKEND_IMAGE }} + ${{ env.BACKEND_IMAGE_LATEST }} + cache-from: type=registry,ref=${{ env.IMAGE_REPOSITORY }}:backend-cache + cache-to: type=registry,ref=${{ env.IMAGE_REPOSITORY }}:backend-cache,image-manifest=true,oci-mediatypes=true,mode=max + + - name: Build frontend image ${{ env.FRONTEND_IMAGE }} + uses: docker/build-push-action@v6 + with: + context: . + file: docker/frontend.dockerfile + pull: true + push: true + tags: | + ${{ env.FRONTEND_IMAGE }} + ${{ env.FRONTEND_IMAGE_LATEST }} + build-args: | + APP_VERSION=${{ github.ref_name }} + cache-from: type=registry,ref=${{ env.IMAGE_REPOSITORY }}:frontend-cache + cache-to: type=registry,ref=${{ env.IMAGE_REPOSITORY }}:frontend-cache,image-manifest=true,oci-mediatypes=true,mode=max + + - name: Substitute web task definition variables + run: | + envsubst < .github/${{ env.TASK_DEFINITION_TEMPLATE_PATH }}/web.yml > web.yml + env: + ROLE_ARN: ${{ env.ROLE_ARN }} + EFS_FILESYSTEM_ID: ${{ env.EFS_FILESYSTEM_ID }} + EFS_ACCESS_POINT_ID: ${{ env.EFS_ACCESS_POINT_ID }} + FRONTEND_CONTAINER_PORT: ${{ env.FRONTEND_CONTAINER_PORT }} + BACKEND_CONTAINER_PORT: ${{ env.BACKEND_CONTAINER_PORT }} + FRONTEND_IMAGE_LATEST: ${{ env.FRONTEND_IMAGE_LATEST }} + BACKEND_IMAGE_LATEST: ${{ env.BACKEND_IMAGE_LATEST }} + AWS_REGION: ${{ env.AWS_REGION }} + TIER: ${{ env.TIER }} + APP: ${{ env.APP }} + ENVIRONMENT_TIER: ${{ env.ENVIRONMENT_TIER }} + TZ: ${{ env.TZ }} + + - name: Print rendered web task definition + run: cat web.yml + + - name: Register web task definition + id: register-web + run: | + arn=$(aws ecs register-task-definition --cli-input-yaml file://web.yml --query "taskDefinition.taskDefinitionArn" --output text) + echo "WEB_TASK_DEF_ARN=$arn" >> $GITHUB_ENV + + - name: Update web service + run: | + aws ecs update-service \ + --cluster ${{ env.ECS_CLUSTER }} \ + --service ${{ env.ECS_WEB_SERVICE }} \ + --task-definition $WEB_TASK_DEF_ARN \ + --desired-count 1 \ + --propagate-tags TASK_DEFINITION \ + --force-new-deployment + + # - name: Wait for service stability + # run: | + # aws ecs wait services-stable \ + # --cluster ${{ env.ECS_CLUSTER }} \ + # --services ${{ env.ECS_WEB_SERVICE }} + + - name: Substitute worker task definition variables + run: | + envsubst < .github/${{ env.TASK_DEFINITION_TEMPLATE_PATH }}/worker.yml > worker.yml + env: + ECS_WORKER_TASK: ${{ env.ECS_WORKER_TASK }} + ECS_WORKER_TASK_CPU_UNITS: ${{ env.ECS_WORKER_TASK_CPU_UNITS }} + ECS_WORKER_TASK_MEMORY_UNITS: ${{ env.ECS_WORKER_TASK_MEMORY_UNITS }} + ROLE_ARN: ${{ env.ROLE_ARN }} + EFS_FILESYSTEM_ID: ${{ env.EFS_FILESYSTEM_ID }} + EFS_ACCESS_POINT_ID: ${{ env.EFS_ACCESS_POINT_ID }} + BACKEND_CONTAINER_PORT: ${{ env.BACKEND_CONTAINER_PORT }} + BACKEND_IMAGE_LATEST: ${{ env.BACKEND_IMAGE_LATEST }} + AWS_REGION: ${{ env.AWS_REGION }} + TIER: ${{ env.TIER }} + APP: ${{ env.APP }} + ENVIRONMENT_TIER: ${{ env.ENVIRONMENT_TIER }} + TZ: ${{ env.TZ }} + + - name: Print rendered worker task definition + run: cat worker.yml + + - name: Register worker task definition + run: aws ecs register-task-definition --cli-input-yaml file://worker.yml + + - name: Remove old task definitions + run: | + for family in "$ECS_WEB_TASK" "$ECS_WORKER_TASK"; do + echo "Pruning old revisions for family: $family" + aws ecs list-task-definitions --family-prefix "$family" --sort DESC --query 'taskDefinitionArns[3:]' --output text \ + | xargs -n1 -r aws ecs deregister-task-definition --task-definition + done \ No newline at end of file