Make DBs run on ramdisk and optimize rspec parallelism

Since in a dev scenario we dont care about data loss we can speed up the
tests majorly by running the DBs on a ramdisk.
Also since the DB consumes CPU during the test its not good to start
rspec executes equal to the number of CPU cores since the sume of load
excedes that by a lot. So we leave 20% spare for the DB to have less
CPU preemtive scheduling and all in all finish the rspecs faster.

An environment variable was introduced PARALLEL_TEST_PROCESSORS_MULTIPLE
that can be set to any floating point value and multiplies the CPU core
count times PARALLEL_TEST_PROCESSORS_MULTIPLE. It defaults to 0.8 and
rounds the CPU count up to the next integer so it cannot be less than 1
ever.

Apply DB performance optimizations also for github actions

Author: FloThinksPi

.devcontainer/devcontainer.json

@@ -20,7 +20,7 @@
         "nginx"
     ],
     "workspaceFolder": "/workspace",
-    "onCreateCommand": "bash .devcontainer/scripts/codespaces_start.sh",
+    "onCreateCommand": "bash .devcontainer/scripts/startup.sh",
     "customizations": {
         // Configure properties specific to VS Code.
         "vscode": {

.devcontainer/scripts/codespaces_start.sh renamed to .devcontainer/scripts/startup.sh

@@ -19,6 +19,7 @@ setupPostgres () {
     echo "Postgres is up."
 
     # Parallel Test DBs
+    export PARALLEL_TEST_PROCESSORS=`ruby -e 'require "etc"; puts (Etc.nprocessors * (ENV["PARALLEL_TEST_PROCESSORS_MULTIPLE"] || 0.8).to_f).ceil'`
     DB="postgres" POSTGRES_CONNECTION_PREFIX="postgres://postgres:supersecret@localhost:5432" bundle exec rake db:pick db:parallel:recreate
     # Sequential Test DBs
     export PGPASSWORD=supersecret
@@ -37,6 +38,7 @@ setupMariadb () {
     echo "MySQL is up."
 
     # Parallel Test DBs
+    export PARALLEL_TEST_PROCESSORS=`ruby -e 'require "etc"; puts (Etc.nprocessors * (ENV["PARALLEL_TEST_PROCESSORS_MULTIPLE"] || 0.8).to_f).ceil'`
     DB="mysql" MYSQL_CONNECTION_PREFIX="mysql2://root:[email protected]:3306" bundle exec rake db:pick db:parallel:recreate
     # Sequential Test DBs
     mysql -h 127.0.0.1 -u root -psupersecret -e "CREATE DATABASE IF NOT EXISTS cc_test; CREATE DATABASE IF NOT EXISTS diego; CREATE DATABASE IF NOT EXISTS locket;"
@@ -54,9 +56,6 @@ POSTGRES_PID=$!
 setupMariadb &
 MARIADB_PID=$!
 
-wait $POSTGRES_PID
-wait $MARIADB_PID
-
 # CC config
 mkdir -p tmp
 cp -a config/cloud_controller.yml tmp/cloud_controller.yml
@@ -120,4 +119,5 @@ yq -i e '.diego.bbs.ca_file="spec/fixtures/certs/bbs_ca.crt"' tmp/cloud_controll
 
 yq -i e '.packages.max_package_size=2147483648' tmp/cloud_controller.yml
 
-# Exit if any error happened
+wait $POSTGRES_PID
+wait $MARIADB_PID

.github/workflows/unit_tests.yml

@@ -54,12 +54,24 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          --tmpfs /var/lib/postgresql/data
+          --shm-size=256m
         ports:
           - 5432:5432
     steps:
     - uses: hmarr/debug-action@v3
     - uses: actions/checkout@v4
     - uses: ./.github/workflows/composite/setup
+    - name: Configure PostgreSQL for performance
+      run: |
+        sudo apt-get update && sudo apt-get install -y postgresql-client
+        PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET fsync = off;"
+        PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET synchronous_commit = off;"
+        PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET full_page_writes = off;"
+        PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET checkpoint_completion_target = 0.9;"
+        PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET wal_buffers = '16MB';"
+        PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET shared_buffers = '256MB';"
+        PGPASSWORD=rootpassword psql -h localhost -U postgres -c "SELECT pg_reload_conf();"
     - name: Run tests
       run: DB=postgres POSTGRES_CONNECTION_PREFIX="postgres://postgres:rootpassword@localhost:5432" bundle exec rake spec
     - uses: ravsamhq/notify-slack-action@v2
@@ -83,13 +95,24 @@ jobs:
         env:
           MYSQL_DATABASE: cc_test
           MYSQL_ROOT_PASSWORD: password
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+          --tmpfs /var/lib/mysql
+          --tmpfs /tmp
         ports:
           - 3306:3306
     steps:
     - uses: hmarr/debug-action@v3
     - uses: actions/checkout@v4
     - uses: ./.github/workflows/composite/setup
+    - name: Configure MySQL for performance
+      run: |
+        sudo apt-get update && sudo apt-get install -y mysql-client
+        mysql -h 127.0.0.1 -u root -ppassword -e "SET GLOBAL innodb_flush_log_at_trx_commit = 0;"
+        mysql -h 127.0.0.1 -u root -ppassword -e "SET GLOBAL innodb_buffer_pool_size = 268435456;" 
     - name: Run tests
       run: DB=mysql MYSQL_CONNECTION_PREFIX="mysql2://root:[email protected]:3306" bundle exec rake spec
     - uses: ravsamhq/notify-slack-action@v2
@@ -98,4 +121,4 @@ jobs:
         status: ${{ job.status }}
         notify_when: 'failure' # default is 'success,failure,warnings'
       env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required

.github/workflows/unit_tests_backwards_compatibility.yml

@@ -4,7 +4,6 @@ concurrency:
   cancel-in-progress: true
 on:
   workflow_dispatch:
-    description: "This action tests backwards compatibility when db migrations are introduced. It tests database schema at new code(old_cc_ref) with unittests running old code(new_cc_ref) "
     inputs:
       old_cc_ref:
         description: 'Old Version of CC_NG that the backwards compatibility should be checked against'
@@ -44,6 +43,8 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          --tmpfs /var/lib/postgresql/data
+          --shm-size=256m
         ports:
           - 5432:5432
     steps:
@@ -63,6 +64,16 @@ jobs:
             )}}
       - name: Setup Environment
         uses: ./.github/workflows/composite/setup
+      - name: Configure PostgreSQL for performance
+        run: |
+          sudo apt-get update && sudo apt-get install -y postgresql-client
+          PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET fsync = off;"
+          PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET synchronous_commit = off;"
+          PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET full_page_writes = off;"
+          PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET checkpoint_completion_target = 0.9;"
+          PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET wal_buffers = '16MB';"
+          PGPASSWORD=rootpassword psql -h localhost -U postgres -c "ALTER SYSTEM SET shared_buffers = '256MB';"
+          PGPASSWORD=rootpassword psql -h localhost -U postgres -c "SELECT pg_reload_conf();"
       - name: Migrate Database
         run: DB=postgres POSTGRES_CONNECTION_PREFIX="postgres://postgres:rootpassword@localhost:5432" bundle exec rake db:parallel:recreate db:parallel:migrate
       - name: Checkout code to run the unit tests with
@@ -92,7 +103,13 @@ jobs:
         env:
           MYSQL_DATABASE: cc_test
           MYSQL_ROOT_PASSWORD: password
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+          --tmpfs /var/lib/mysql
+          --tmpfs /tmp
         ports:
           - 3306:3306
     steps:
@@ -112,6 +129,11 @@ jobs:
             )}}
       - name: Setup Environment
         uses: ./.github/workflows/composite/setup
+      - name: Configure MySQL for performance
+        run: |
+          sudo apt-get update && sudo apt-get install -y mysql-client
+          mysql -h 127.0.0.1 -u root -ppassword -e "SET GLOBAL innodb_flush_log_at_trx_commit = 0;"
+          mysql -h 127.0.0.1 -u root -ppassword -e "SET GLOBAL innodb_buffer_pool_size = 268435456;"
       - name: Migrate Database
         run: DB=mysql MYSQL_CONNECTION_PREFIX="mysql2://root:[email protected]:3306" bundle exec rake db:parallel:recreate db:parallel:migrate
       - name: Checkout code to run the unit tests with
@@ -125,4 +147,4 @@ jobs:
       - name: Setup Environment
         uses: ./.github/workflows/composite/setup
       - name: Run tests
-        run: DB=mysql MYSQL_CONNECTION_PREFIX="mysql2://root:[email protected]:3306" bundle exec rake spec:without_migrate
+        run: DB=mysql MYSQL_CONNECTION_PREFIX="mysql2://root:[email protected]:3306" bundle exec rake spec:without_migrate

devenv.sh

@@ -18,27 +18,27 @@ help_command() {
 
 # Create a clean development environment
 create_command(){
-  docker-compose -p "" down
+  docker compose -p "" down
   docker buildx bake -f docker-compose.yml &
-  docker-compose -p "" pull &
+  docker compose -p "" pull &
   wait $(jobs -p)
-  docker-compose -p "" up -d --build
-  ./.devcontainer/scripts/setupDevelopmentEnvironment.sh
+  docker compose -p "" up -d --build
+  ./.devcontainer/scripts/startup.sh
 }
 
 # Start containers
 start_command(){
-  docker-compose -p "" start
+  docker compose -p "" start
 }
 
 # Stop containers
 stop_command(){
-  docker-compose -p "" stop
+  docker compose -p "" stop
 }
 
 # Remove containers
 destroy_command(){
-  docker-compose -p "" down
+  docker compose -p "" down
 }
 
 # Call Setup IDEs Script
@@ -72,7 +72,7 @@ fi
 # Check Prerequisites
 export should_exit=0
 # Check Path Exists
-for p in docker docker-compose ruby bundle mysql psql yq; do
+for p in docker ruby bundle mysql psql yq; do
     if ! command -v "${p}" >/dev/null 2>&1; then
       echo "Error: Dependency \"$p\" is not installed" && export should_exit=1
     fi

docker-compose.yml

@@ -7,10 +7,26 @@ services:
     environment:
       POSTGRES_PASSWORD: supersecret
     command:
-      # logs every statement sent to the server. Slow, but helpful. Fills up disk Quickly !
       - "postgres"
-      # - "-c"
-      # - "log_statement=all"
+      - "-c"
+      - "fsync=off"
+      - "-c"
+      - "synchronous_commit=off"
+      - "-c"
+      - "full_page_writes=off"
+      - "-c"
+      - "log_statement=all"
+      - "-c"
+      - "checkpoint_segments=32"
+      - "-c"
+      - "checkpoint_completion_target=0.9"
+      - "-c"
+      - "wal_buffers=16MB"
+      - "-c"
+      - "shared_buffers=256MB"
+    tmpfs:
+      - /var/lib/postgresql/data
+    shm_size: 256m
     ports:
       - "127.0.0.1:5432:5432"
     networks:
@@ -28,6 +44,18 @@ services:
     image: mysql:8.2
     environment:
       MYSQL_ROOT_PASSWORD: supersecret
+    command:
+      - "--innodb-flush-method=nosync"
+      - "--innodb-flush-log-at-trx-commit=0"
+      - "--innodb-doublewrite=0"
+      - "--innodb-buffer-pool-size=256M"
+      - "--innodb-log-buffer-size=16M"
+      - "--innodb-log-file-size=64M"
+      - "--skip-innodb-checksums"
+      - "--skip-sync-frm"
+    tmpfs:
+      - /var/lib/mysql
+      - /tmp
     ports:
       - "127.0.0.1:3306:3306"
     healthcheck:

lib/tasks/spec.rake

@@ -2,7 +2,9 @@ desc 'Runs all specs'
 task spec: 'spec:all'
 
 namespace :spec do
-  task all: ['db:pick', 'db:parallel:recreate'] do
+  task all: ['db:pick'] do
+    ENV['PARALLEL_TEST_PROCESSORS'] = (Etc.nprocessors * (ENV['PARALLEL_TEST_PROCESSORS_MULTIPLE'] || 0.8).to_f).ceil.to_s
+    Rake::Task['db:parallel:recreate'].invoke
     if ARGV[1]
       run_specs(ARGV[1])
     else
@@ -44,6 +46,7 @@ namespace :spec do
   def run_specs_parallel(path, env_vars='')
     command = <<~CMD
       #{env_vars} bundle exec parallel_rspec \
+      -n `ruby -e 'require "etc"; puts (Etc.nprocessors * (ENV["PARALLEL_TEST_PROCESSORS_MULTIPLE"] || 0.8).to_f).ceil'` \
       --test-options '--order rand' \
       --single spec/integration/ \
       --single spec/acceptance/ \