Improve spockbench CI workflow and test execution

Update GitHub Actions workflow to run spockbench tests in parallel across
all three nodes with proper synchronization and cleanup. Mount test scripts
as volumes to enable rapid iteration without rebuilding containers.

Changes:
- Split workflow into separate steps for cluster startup, test execution,
  validation, and cleanup with appropriate timeouts
- Run tests on all three nodes (n1, n2, n3) in parallel using background
  processes with proper PID tracking
- Add always-run cleanup step to ensure docker-compose down executes
- Mount tests directory as volume in docker-compose for faster iteration
- Update spockbench branch to delta-apply-update and fix installation
- Simplify run-tests.sh to use environment variables instead of arguments
- Use delta_apply function with proper DDL replication settings
- Fix psql authentication by relying on trust configuration
- Use default repset instead of demo_replication_set for consistency

Author: danolivo

.github/workflows/spockbench.yml

@@ -103,14 +103,37 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
-      - name: Start docker
+      - name: Start docker cluster
         run: |
           cd ${GITHUB_WORKSPACE}/tests/docker/
           echo PG_VER=${{ matrix.pgver }} >> pgedge.env
-          docker compose up
+          env BASE_IMAGE=ghcr.io/pgedge/base-debug-image:latest \
+            docker-compose up --wait -d --build
+        timeout-minutes: 20
+
+      - name: Run tests on all nodes
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          docker-compose exec -T pgedge-n1 bash -c "~/tests/run-tests.sh" &
+          PID1=$!
+          docker-compose exec -T pgedge-n2 bash -c "~/tests/run-tests.sh" &
+          PID2=$!
+          docker-compose exec -T pgedge-n3 bash -c "~/tests/run-tests.sh" &
+          PID3=$!
+          wait $PID1
+          wait $PID2
+          wait $PID3
+        timeout-minutes: 30
 
       - name: Check spockbench output
+        if: ${{ always() }}
         run: |
           cd ${GITHUB_WORKSPACE}/tests/docker
-          ./check-outputs.sh
+          ./check-outputs.sh || true
+
+      - name: Cleanup docker
+        if: ${{ always() }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          docker-compose down || true
 

tests/docker/Dockerfile-step-1.el9

@@ -126,7 +126,7 @@ RUN set -eux && \
 # Clone spockbench for testing workloads
 # TODO: Pin to specific tag/commit for reproducibility
 RUN set -eux && \
-    git clone --branch master --depth 1 \
+    git clone --branch delta-apply-update --depth 1 \
         https://github.com/pgEdge/spockbench.git /home/pgedge/spockbench && \
 	echo "export SPOCKBENCH_SOURCE_DIR=/home/pgedge/spockbench" >> /home/pgedge/.bashrc
 
@@ -214,7 +214,7 @@ RUN set -eux && \
 
 RUN set -eux && \
     cd /home/pgedge/spockbench && \
-    python3 setup.py install --user && \
+    sudo python3 setup.py install && \
     echo "Spockbench installation complete"
 
 # ==============================================================================

tests/docker/docker-compose.yml

@@ -29,6 +29,7 @@ services:
     # Mount volume for core dumps (persists across container restarts)
     volumes:
       - ./cores/n1:/cores
+      - .:/home/pgedge/tests
     healthcheck:
       test: ["CMD", "pg_isready", "-h", "/tmp", "-U", "admin"]
       interval: 10s
@@ -69,6 +70,7 @@ services:
     # Mount volume for core dumps (persists across container restarts)
     volumes:
       - ./cores/n2:/cores
+      - .:/home/pgedge/tests
     healthcheck:
       test: ["CMD", "pg_isready", "-h", "/tmp", "-U", "admin"]
       interval: 10s
@@ -109,6 +111,7 @@ services:
     # Mount volume for core dumps (persists across container restarts)
     volumes:
       - ./cores/n3:/cores
+      - .:/home/pgedge/tests
     healthcheck:
       test: ["CMD", "pg_isready", "-h", "/tmp", "-U", "admin"]
       interval: 10s

tests/docker/run-tests.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
 
+source "${HOME}/.bashrc"
+
 #set -euo pipefail
 
-peer_names=$1
+IFS=',' read -r -a peer_names <<< "$PEER_NAMES"
 
 #========== Exception Log tests ==========
 
@@ -19,7 +21,7 @@ peer_names=$1
 # ----
 if [[ $(hostname) == "n1" ]];
 then
-  psql -U $DBUSER -d $DBNAME -h /tmp <<_EOF_
+  psql -h /tmp <<_EOF_
   CREATE TABLE t4 (
     id		integer PRIMARY KEY,
     data	text
@@ -29,28 +31,28 @@ then
   INSERT INTO t4 VALUES (3, 'missing row on DELETE');
 
   SELECT spock.repset_add_table(
-    set_name := 'demo_replication_set',
+    set_name := 'default',
     relation := 't4'
   );
 _EOF_
 
   # ----
   # Create table and test data on n2
   # ----
-  PGPASSWORD=$DBPASSWD psql -U $DBUSER -d $DBNAME -h ${peer_names[0]} <<_EOF_
+  psql -h ${peer_names[0]} <<_EOF_
   CREATE TABLE t4 (
     id		integer PRIMARY KEY,
     data	text
   );
 
   INSERT INTO t4 VALUES (1, 'duplicate key on INSERT');
   SELECT spock.repset_add_table(
-    set_name := 'demo_replication_set',
+    set_name := 'default',
     relation := 't4'
   );
 _EOF_
 
-  psql -U $DBUSER -d $DBNAME -h /tmp <<_EOF_
+  psql -h /tmp <<_EOF_
   INSERT INTO t4 VALUES (1, 'trigger duplicate key');
   UPDATE t4 SET data = 'trigger missing key on UPDATE' WHERE id = 2;
   DELETE FROM t4 WHERE id = 3; -- trigger missing key on DELETE
@@ -59,7 +61,7 @@ _EOF_
   echo "Waiting for apply worker timeouts..."
   sleep 5
   echo "Checking the exception table now..."
-  elog_entries=$(PGPASSWORD=$DBPASSWD psql -A -t -U $DBUSER -d $DBNAME -h ${peer_names[0]} -c "
+  elog_entries=$(psql -A -t -h ${peer_names[0]} -c "
   	SELECT count(*)
 	FROM spock.exception_log e
 	JOIN spock.node n
@@ -71,13 +73,13 @@ _EOF_
 
   if [ "$elog_entries" -ne 1 ];
   then
-	  PGPASSWORD=$DBPASSWD psql -U $DBUSER -d $DBNAME -h ${peer_names[0]} -c "select * from spock.exception_log;"
+	  psql -h ${peer_names[0]} -c "select * from spock.exception_log;"
 	  echo "Did not find an exception log entry. Exiting..."
 	  exit 1
   fi
 
 
-  resolution_check=$(PGPASSWORD=$DBPASSWD psql -X -A -t -U $DBUSER -d $DBNAME -h ${peer_names[0]} -c " SELECT conflict_type FROM spock.resolutions WHERE relname = 'public.t4'")
+  resolution_check=$(psql -X -A -t -h ${peer_names[0]} -c " SELECT conflict_type FROM spock.resolutions WHERE relname = 'public.t4'")
 
   insert_exists_count=$(echo "$resolution_check" | grep -c 'insert_exists')
   delete_delete_count=$(echo "$resolution_check" | grep -c 'delete_delete')
@@ -86,22 +88,31 @@ _EOF_
   then
     echo "PASS: Found both insert_exists and delete_delete for public.t4"
   else
-    PGPASSWORD=$DBPASSWD psql -U $DBUSER -d $DBNAME -h ${peer_names[0]} -c "select * from spock.resolutions where relname = 'public.t4'"
+    psql -h ${peer_names[0]} -c "select * from spock.resolutions where relname = 'public.t4'"
     echo "FAIL: Resolution entries for public.t4 are incorrect"
     echo "Resolutions check=$resolution_check"
     echo "Found: insert_exists=$insert_exists_count, delete_delete=$delete_delete_count"
     exit 1
   fi
 fi
 
-spockbench -h /tmp -i -s $SCALEFACTOR demo
-psql -U admin -h /tmp -d demo -c "alter table pgbench_accounts alter column abalance set(log_old_value=true, delta_apply_function=spock.delta_apply);"
-psql -U admin -h /tmp -d demo -c "alter table pgbench_branches alter column bbalance set(log_old_value=true, delta_apply_function=spock.delta_apply);"
-psql -U admin -h /tmp -d demo -c "alter table pgbench_tellers alter column tbalance set(log_old_value=true, delta_apply_function=spock.delta_apply);"
+spockbench -h /tmp -i -s $SCALEFACTOR $PGDATABASE
+
+psql -h /tmp <<_EOF_
+  SET spock.enable_ddl_replication = 'on';
+  SET spock.include_ddl_repset = 'on';
 
-psql -U admin -h /tmp -d demo -c "select spock.repset_add_all_tables('demo_replication_set', '{public}');"
+  SELECT spock.repset_add_all_tables('default', '{public}');
+
+  ALTER TABLE pgbench_accounts ALTER COLUMN abalance SET NOT NULL;
+  ALTER TABLE pgbench_branches ALTER COLUMN bbalance SET NOT NULL;
+  ALTER TABLE pgbench_tellers ALTER COLUMN tbalance SET NOT NULL;
+  SELECT spock.delta_apply('pgbench_accounts', 'abalance', false);
+  SELECT spock.delta_apply('pgbench_branches', 'bbalance', false);
+  SELECT spock.delta_apply('pgbench_tellers', 'tbalance', false);
+_EOF_
 
 # ==========Spockbench tests ==========
-spockbench -h /tmp --spock-num-nodes=3 --spock-node=${HOSTNAME:0-1} -s $SCALEFACTOR -T $RUNTIME -R $RATE -P 5 -j $THREADS -c $CONNECTIONS -n --spock-tx-mix=550,225,225 -U admin demo
-spockbench-check -U admin demo > /home/pgedge/spock/spockbench-$HOSTNAME.out
+spockbench -h /tmp --spock-num-nodes=3 --spock-node=${HOSTNAME:0-1} -s $SCALEFACTOR -T $RUNTIME -R $RATE -P 5 -j $THREADS -c $CONNECTIONS -n --spock-tx-mix=550,225,225 -U $PGUSER $PGDATABASE
+spockbench-check -U $PGUSER $PGDATABASE > /home/pgedge/spock/spockbench-$HOSTNAME.out
 grep -q "ERROR" /home/pgedge/spock/spockbench-*.out && exit 1 || exit 0