Make DBs run on ramdisk and optimize rspec parallelism

Since in a dev scenario we dont care about data loss we can speed up the
tests majorly by running the DBs on a ramdisk.
Also since the DB consumes CPU during the test its not good to start
rspec executes equal to the number of CPU cores since the sume of load
excedes that by a lot. So we leave 20% spare for the DB to have less
CPU preemtive scheduling and all in all finish the rspecs faster.

An environment variable was introduced PARALLEL_TEST_PROCESSORS_MULTIPLE
that can be set to any floating point value and multiplies the CPU core
count times PARALLEL_TEST_PROCESSORS_MULTIPLE. It defaults to 0.8 and
rounds the CPU count up to the next integer so it cannot be less than 1
ever.

Author: FloThinksPi

.devcontainer/devcontainer.json

@@ -20,7 +20,7 @@
         "nginx"
     ],
     "workspaceFolder": "/workspace",
-    "onCreateCommand": "bash .devcontainer/scripts/codespaces_start.sh",
+    "onCreateCommand": "bash .devcontainer/scripts/startup.sh",
     "customizations": {
         // Configure properties specific to VS Code.
         "vscode": {

.devcontainer/scripts/codespaces_start.sh renamed to .devcontainer/scripts/startup.sh

@@ -19,6 +19,7 @@ setupPostgres () {
     echo "Postgres is up."
 
     # Parallel Test DBs
+    export PARALLEL_TEST_PROCESSORS=`ruby -e 'require "etc"; puts (Etc.nprocessors * (ENV["PARALLEL_TEST_PROCESSORS_MULTIPLE"] || 0.8).to_f).ceil'`
     DB="postgres" POSTGRES_CONNECTION_PREFIX="postgres://postgres:supersecret@localhost:5432" bundle exec rake db:pick db:parallel:recreate
     # Sequential Test DBs
     export PGPASSWORD=supersecret
@@ -37,6 +38,7 @@ setupMariadb () {
     echo "MySQL is up."
 
     # Parallel Test DBs
+    export PARALLEL_TEST_PROCESSORS=`ruby -e 'require "etc"; puts (Etc.nprocessors * (ENV["PARALLEL_TEST_PROCESSORS_MULTIPLE"] || 0.8).to_f).ceil'`
     DB="mysql" MYSQL_CONNECTION_PREFIX="mysql2://root:[email protected]:3306" bundle exec rake db:pick db:parallel:recreate
     # Sequential Test DBs
     mysql -h 127.0.0.1 -u root -psupersecret -e "CREATE DATABASE IF NOT EXISTS cc_test; CREATE DATABASE IF NOT EXISTS diego; CREATE DATABASE IF NOT EXISTS locket;"
@@ -54,9 +56,6 @@ POSTGRES_PID=$!
 setupMariadb &
 MARIADB_PID=$!
 
-wait $POSTGRES_PID
-wait $MARIADB_PID
-
 # CC config
 mkdir -p tmp
 cp -a config/cloud_controller.yml tmp/cloud_controller.yml
@@ -120,4 +119,5 @@ yq -i e '.diego.bbs.ca_file="spec/fixtures/certs/bbs_ca.crt"' tmp/cloud_controll
 
 yq -i e '.packages.max_package_size=2147483648' tmp/cloud_controller.yml
 
-# Exit if any error happened
+wait $POSTGRES_PID
+wait $MARIADB_PID

devenv.sh

@@ -18,27 +18,27 @@ help_command() {
 
 # Create a clean development environment
 create_command(){
-  docker-compose -p "" down
+  docker compose -p "" down
   docker buildx bake -f docker-compose.yml &
-  docker-compose -p "" pull &
+  docker compose -p "" pull &
   wait $(jobs -p)
-  docker-compose -p "" up -d --build
-  ./.devcontainer/scripts/setupDevelopmentEnvironment.sh
+  docker compose -p "" up -d --build
+  ./.devcontainer/scripts/startup.sh
 }
 
 # Start containers
 start_command(){
-  docker-compose -p "" start
+  docker compose -p "" start
 }
 
 # Stop containers
 stop_command(){
-  docker-compose -p "" stop
+  docker compose -p "" stop
 }
 
 # Remove containers
 destroy_command(){
-  docker-compose -p "" down
+  docker compose -p "" down
 }
 
 # Call Setup IDEs Script
@@ -72,7 +72,7 @@ fi
 # Check Prerequisites
 export should_exit=0
 # Check Path Exists
-for p in docker docker-compose ruby bundle mysql psql yq; do
+for p in docker ruby bundle mysql psql yq; do
     if ! command -v "${p}" >/dev/null 2>&1; then
       echo "Error: Dependency \"$p\" is not installed" && export should_exit=1
     fi

docker-compose.yml

@@ -7,10 +7,17 @@ services:
     environment:
       POSTGRES_PASSWORD: supersecret
     command:
-      # logs every statement sent to the server. Slow, but helpful. Fills up disk Quickly !
       - "postgres"
-      # - "-c"
-      # - "log_statement=all"
+      - "-c"
+      - "fsync=off"
+      - "-c"
+      - "synchronous_commit=off"
+      - "-c"
+      - "full_page_writes=off"
+      - "-c"
+      - "log_statement=all"
+    tmpfs:
+      - /var/lib/postgresql/data
     ports:
       - "127.0.0.1:5432:5432"
     networks:
@@ -28,6 +35,12 @@ services:
     image: mysql:8.2
     environment:
       MYSQL_ROOT_PASSWORD: supersecret
+    command:
+      - "--innodb-flush-method=nosync"
+      - "--innodb-flush-log-at-trx-commit=0"
+      - "--innodb-doublewrite=0"
+    tmpfs:
+      - /var/lib/mysql
     ports:
       - "127.0.0.1:3306:3306"
     healthcheck:

lib/tasks/spec.rake

@@ -2,7 +2,9 @@ desc 'Runs all specs'
 task spec: 'spec:all'
 
 namespace :spec do
-  task all: ['db:pick', 'db:parallel:recreate'] do
+  task all: ['db:pick'] do
+    ENV['PARALLEL_TEST_PROCESSORS'] = (Etc.nprocessors * (ENV['PARALLEL_TEST_PROCESSORS_MULTIPLE'] || 0.8).to_f).ceil.to_s
+    Rake::Task['db:parallel:recreate'].invoke
     if ARGV[1]
       run_specs(ARGV[1])
     else
@@ -44,6 +46,7 @@ namespace :spec do
   def run_specs_parallel(path, env_vars='')
     command = <<~CMD
       #{env_vars} bundle exec parallel_rspec \
+      -n `ruby -e 'require "etc"; puts (Etc.nprocessors * (ENV["PARALLEL_TEST_PROCESSORS_MULTIPLE"] || 0.8).to_f).ceil'` \
       --test-options '--order rand' \
       --single spec/integration/ \
       --single spec/acceptance/ \