DPC-5117 solid queue as separate service (#2915)

## 🎫 Ticket

https://jira.cms.gov/browse/DPC-5117

## 🛠 Changes

- Solid Queue initialization moved from puma to Proc files
- Async portal added to check_healthy on deploy
- Check healthy script unused parameter removed; check for target
removed for async
- Docker entrypoint includes separate async section

## ℹ️ Context

We want to separate our rails and SolidQueue processes for the portal to
reduce resource contention.
Locally, we continue to run in same container.

Coupled with changes in
[dpc-ops](CMSgov/dpc-ops#919).

## 🧪 Validation

Successfully deployed to test:
https://github.com/CMSgov/dpc-app/actions/runs/22322110206

Author: jdettmannnava

.github/workflows/check_healthy.yml

@@ -17,22 +17,25 @@ jobs:
       matrix:
         include:
           - service: api
-            script_params: api-1 api api frontend v9
+            script_params: api-1 api frontend v9
             skip: false
           - service: attribution
-            script_params: attribution attribution attribution backend v9
+            script_params: attribution attribution backend v9
             skip: false
           - service: aggregation
-            script_params: aggregation aggregation aggregation backend v9
+            script_params: aggregation aggregation backend v9
             skip: false
           - service: web
-            script_params: web-v2-1 web web frontend v9
+            script_params: web-v2-1 web frontend v9
             skip: ${{ inputs.env == 'prod' }}
           - service: admin
-            script_params: web-admin-1 web-admin web-admin frontend v9
+            script_params: web-admin-1 web-admin frontend v9
             skip: ${{ inputs.env == 'prod' }}
-          - service: portal
-            script_params: web-portal-1 dpc-portal web-portal frontend v9
+          - service: web-portal
+            script_params: web-portal-1 web-portal frontend v9
+            skip: ${{ inputs.env == 'prod' || inputs.env == 'sandbox' }}
+          - service: async-portal
+            script_params: none async-portal frontend v9
             skip: ${{ inputs.env == 'prod' || inputs.env == 'sandbox' }}
     steps:
       - name: "Checkout code"

dpc-portal/Procfile.dev

@@ -1,2 +1,3 @@
 web: bundle exec rails server -b 0.0.0.0 -p 3100
+solidqueue: bundle exec rails solid_queue:start
 cpigw: env PORT=4567 ruby spec/support/fake_cpi_gateway.rb

dpc-portal/Procfile.nonprod_async

@@ -0,0 +1,2 @@
+solidqueue: bundle exec rails solid_queue:start
+cpigw: env PORT=4567 ruby spec/support/fake_cpi_gateway.rb

dpc-portal/bin/nonprod_async

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+echo "Testing removal of foreman log prefix (should be json)"
+export LOG_STR='06:21:25 web.1   | {"level": "debug", "time": "blahblahblah"}'
+export EVERYTHING_AFTER_PIPE_REGEX='s/[^|]*|//'
+echo $LOG_STR | sed $EVERYTHING_AFTER_PIPE_REGEX
+
+echo "Starting Procfile.nonprod_async"
+exec foreman start -f Procfile.nonprod_async "$@" | sed $EVERYTHING_AFTER_PIPE_REGEX

dpc-portal/config/application.rb

@@ -27,6 +27,7 @@ class Application < Rails::Application
 
     config.active_support.to_time_preserves_timezone = :zone
     config.active_job.queue_adapter = :solid_queue
+    config.solid_queue.supervisor_pidfile = 'tmp/solid_queue_pidfile'
 
     # Ensure mailer jobs get sent to a specialized queue.
     config.action_mailer.deliver_later_queue_name = "portal"

dpc-portal/config/puma.rb

@@ -43,4 +43,3 @@
 
 # Allow puma to be restarted by `bin/rails restart` command.
 plugin :tmp_restart
-plugin :solid_queue

dpc-portal/docker/entrypoint.sh

@@ -6,6 +6,10 @@ set -e
 if [ -f tmp/pids/server.pid ]; then
   rm tmp/pids/server.pid
 fi
+if [ -f tmp/solid_queue_pidfile ]; then
+  echo "Removing solid queue pidfile"
+  rm tmp/solid_queue_pidfile
+fi
 
 if [ "$1" == "portal" ]; then
   # Start the database service (and make accessible outside the Docker container)
@@ -24,4 +28,14 @@ if [ "$1" == "portal" ]; then
     echo "Starting in non-production"
     ./bin/nonprod
   fi
+elif [ "$1" == "async" ]; then
+  echo "Starting Solid Queue..."
+  if [[ "$ENV" == "production" ]]; then
+    echo "Starting in production"
+    bundle exec rails solid_queue:start
+  # For local, SolidQueue starts in same container
+  elif [[ "$ENV" != "local" ]]; then
+    echo "Starting in non-production"
+    ./bin/nonprod_async
+  fi
 fi

scripts/check-deployment-status.sh

@@ -4,11 +4,11 @@ set -euxo pipefail
 
 ENV="$1" # env
 TARGET_GROUP="dpc-${ENV}-${2}" # target
-SVC_NAME="dpc-${ENV}-${4}" # service
-CLUSTER_NAME="dpc-${ENV}-${5}" # cluster
-SVC_VERSION="$6" # service version
+SVC_NAME="dpc-${ENV}-${3}" # service
+CLUSTER_NAME="dpc-${ENV}-${4}" # cluster
+SVC_VERSION="$5" # service version
 
-if [ $7 == 'true' ]; then
+if [ $6 == 'true' ]; then
     echo "Skipping $SVC_NAME"
     exit 0
 fi
@@ -80,6 +80,12 @@ echo "ECS service ${SVC_NAME}-${SVC_VERSION} stable"
 echo "---"
 echo "1. Checking for existing Target Group: $TARGET_GROUP"
 
+if [ $2 == "none" ]; then
+  echo "Skipping Target Group Check as no Target Group defined"
+  echo "Deployment successful and application layer confirmed healthy."
+  exit 0
+fi
+
 set +e
 TARGET_GROUP_ARN=$(aws elbv2 describe-target-groups \
   --names "$TARGET_GROUP" \