SPOC-345: Move zodan sync test to separate github action

.github/workflows/zodan_sync.yml

@@ -0,0 +1,77 @@
+# Keep its logic in sync with spockbench.yml!
+
+name: Z0DAN Sync long-lasting test
+run-name: Add new node into highly loaded multi-master configuration
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 22 * * 6'  # Saturday, 22:00 (UTC)
+
+permissions:
+  contents: read
+
+jobs:
+  pull-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        pgver: [15, 16, 17, 18]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout spock
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Add permissions
+        run: |
+          sudo chmod -R a+w ${GITHUB_WORKSPACE}
+
+      - name: Set up Docker
+        # Codacy wants us to use full commit SHA. This is for v3
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
+
+      - name: Set up docker compose
+        # Codacy wants us to use full commit SHA. This is for v1
+        uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 # v1
+        with:
+          version: latest
+
+      - name: Build docker container
+        run: |
+          cd ${GITHUB_WORKSPACE}/
+          ls -la
+          ls -la sql
+          ls -la tests/
+          ls -la tests/tap
+          ls -la tests/tap/t
+          cat tests/tap/t/011_zodan_sync_third.pl
+          docker build \
+            --build-arg PGVER=${{ matrix.pgver }} \
+            -t spock -f tests/docker/Dockerfile-step-1.el9 .
+
+      - name: Run picked TAP tests
+        run: |
+          TAP_CT_NAME="spock-tap-${{ matrix.pgver }}-${{ github.run_id }}-${{ github.run_attempt }}"
+          echo "TAP_CT_NAME=$TAP_CT_NAME" >> "$GITHUB_ENV"
+          docker run --name "$TAP_CT_NAME" -e PGVER=${{ matrix.pgver }} spock /home/pgedge/run-spock-tap.sh "t/011_zodan_sync_third.pl" 10
+        timeout-minutes: 1440
+
+      - name: Collect TAP artifacts (from container)
+        if: ${{ always() }}
+        run: |
+          docker cp "$TAP_CT_NAME":/home/pgedge/spock/ "$GITHUB_WORKSPACE/results" || true
+          docker rm -f "$TAP_CT_NAME" || true
+
+      - name: Upload TAP artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: tap-${{ matrix.pgver }}
+          path: |
+            ${{ github.workspace }}/results
+          if-no-files-found: ignore
+          retention-days: 7

tests/docker/Dockerfile-base.el9

@@ -28,7 +28,7 @@ RUN ssh-keygen -t ed25519 -N "" -f ~/.ssh/id_ed25519 && \
     cat ~/.ssh/*.pub >> ~/.ssh/authorized_keys
 
 
-COPY . /home/pgedge/spock
+RUN mkdir -p /home/pgedge/spock
 RUN chown -R pgedge:pgedge /home/pgedge/spock
 #-----------------------------------------
 USER pgedge

tests/docker/Dockerfile-step-1.el9

@@ -4,6 +4,7 @@ ARG PGVER
 
 ENV PGVER=$PGVER
 
+COPY . /home/pgedge/spock
 WORKDIR /home/pgedge
 
 RUN echo "Determine PostgreSQL tag"
@@ -32,7 +33,7 @@ RUN for patchfile in /home/pgedge/spock/patches/${PGVER}/*; do \
 	done
 
 RUN echo "==========Compiling Modified PostgreSQL=========="
-RUN options="'--prefix=/home/pgedge/pgedge/pg$PGVER' '--disable-rpath' '--with-zstd' '--with-lz4' '--with-icu' '--with-libxslt' '--with-libxml' '--with-uuid=ossp' '--with-gssapi' '--with-ldap' '--with-pam' '--enable-debug' '--enable-dtrace' '--with-llvm' 'LLVM_CONFIG=/usr/bin/llvm-config-64' '--with-openssl' '--with-systemd' '--enable-tap-tests' '--with-python' '--enable-cassert' 'PYTHON=/usr/bin/python3.9' 'BITCODE_CFLAGS=-gdwarf-5 -O0 -fforce-dwarf-frame' 'CFLAGS=-g -O0'" && eval ./configure $options && make -j4 && make install
+RUN options="'--prefix=/home/pgedge/pgedge/pg$PGVER' '--disable-rpath' '--with-zstd' '--with-lz4' '--with-icu' '--with-libxslt' '--with-libxml' '--with-uuid=ossp' '--with-gssapi' '--with-ldap' '--with-pam' '--enable-debug' '--enable-dtrace' '--with-llvm' 'LLVM_CONFIG=/usr/bin/llvm-config-64' '--with-openssl' '--with-systemd' '--enable-tap-tests' '--with-python' '--enable-cassert' 'PYTHON=/usr/bin/python3.9' 'BITCODE_CFLAGS=-gdwarf-5 -O0 -fforce-dwarf-frame' 'CFLAGS=-g -O0'" && eval ./configure $options && make -j4 && make -C contrib -j4 && make install && make -C contrib install
 
 WORKDIR /home/pgedge
 
@@ -46,9 +47,18 @@ RUN . /home/pgedge/.bashrc && export PG_CONFIG=/home/pgedge/pgedge/pg$PGVER/bin/
 RUN echo "==========Built Spock=========="
 
 #-----------------------------------------
-COPY tests/docker/entrypoint.sh tests/docker/run-tests.sh tests/docker/run-spock-regress.sh /home/pgedge
-
-RUN sudo chmod +x /home/pgedge/entrypoint.sh /home/pgedge/run-tests.sh /home/pgedge/run-spock-regress.sh
+COPY \
+	tests/docker/entrypoint.sh \
+	tests/docker/run-tests.sh \
+	tests/docker/run-spock-regress.sh \
+	tests/docker/run-spock-tap.sh \
+	/home/pgedge
+
+RUN sudo chmod +x \
+	/home/pgedge/entrypoint.sh \
+	/home/pgedge/run-tests.sh \
+	/home/pgedge/run-spock-regress.sh \
+	/home/pgedge/run-spock-tap.sh
 
 WORKDIR /home/pgedge/
 USER pgedge

tests/docker/run-spock-tap.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+###
+### Run selected TAP tests iteratively
+###
+
+
+export PG_CONFIG=/home/pgedge/pgedge/pg${PGVER}/bin/pg_config
+export PATH=/home/pgedge/pgedge/pg${PGVER}/bin:$PATH
+export LD_LIBRARY_PATH=/home/pgedge/pgedge/pg${PGVER}/lib/:$LD_LIBRARY_PATH
+export PROVE_TESTS=$proven_tests
+
+# PGVER should be previously set in the environment
+if [ -z "${PGVER}" ]
+then
+	echo "The PGVER environment variable must be set before running this command"
+	exit 1
+fi
+
+proven_tests="$1"
+iterations="$2"
+if [[ -z "$proven_tests" || -z "$iterations" ]]; then
+    echo "Command-line parameters are set incorrectly"
+    exit 1
+fi
+
+cd /home/pgedge/spock/
+
+status=0
+for i in $(seq 1 $iterations); do
+    echo "Iteration $i: running make check..."
+    env PROVE_TESTS="$proven_tests" make check_prove 1>out.txt 2>err.txt
+    status=$?
+    if [ $status -ne 0 ]; then
+        echo "make check failed with status $status on iteration $i"
+        break
+    fi
+done
+
+if [ $status -ne 0 ]
+then
+	echo "Errors in regression checks"
+	exit 1
+fi

tests/tap/schedule

@@ -15,4 +15,18 @@ test: 004_non_default_repset
 test: 008_rmgr
 test: 009_zodan_add_remove_nodes
 test: 010_zodan_add_remove_python
-test: 011_zodan_sync_third
+
+# Tests, consuming too much time to be launched on each check:
+#test: 011_zodan_sync_third
+#
+# Use GitHub Actions to launch them (see workflows/zodan_sync.yml for an example
+# Also, it may be run locally by a bash script like the following:
+#
+# for i in {1..1000}; do
+#    env PROVE_TESTS="t/011_zodan_sync_third.pl" make check_prove 1>out.txt 2>err.txt
+#    status=$?
+#    if [ $status -ne 0 ]; then
+#        echo "make check failed with status $status on iteration $i"
+#        break
+#    fi
+# done

tests/tap/t/011_zodan_sync_third.pl

@@ -24,7 +24,7 @@
 create_cluster(3, 'Create initial 2-node Spock test cluster');
 
 
-my ($ret1, $ret2, $ret3);
+my ($ret1, $ret2, $ret3, $lsn1, $lsn2, $lsn3);
 
 # Get cluster configuration
 my $config = get_test_config();
@@ -264,7 +264,7 @@
 $alive = kill 0, $pid;
 ok($alive eq 1, "pgbench load to N2 still exists");
 
-print STDERR "Kill pgbench process to reduce test time";
+print STDERR "Kill pgbench process to reduce test time\n";
 $pgbench_handle1->pump();
 $pgbench_handle2->pump();
 $pgbench_handle1->kill_kill;
@@ -363,40 +363,30 @@
 print STDERR $pgbench_stdout2;
 print STDERR "##### end of output #####\n";
 
-# We need such a trick: the wait_slot_confirm_lsn routine gets Last Committed
-# LSN position and waits for the confirmations on the remote side. But if there
-# a conflict has happened, feedback will not be sent and we will wait forever.
-psql_or_bail(1, "SELECT spock.sync_event()");
-psql_or_bail(2, "SELECT spock.sync_event()");
-psql_or_bail(3, "SELECT spock.sync_event()");
-
-psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)');
-psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)');
-
-print STDERR "Wait for the end of N3->N1, N3->N2 decoding process that means the actual start of LR\n";
-psql_or_bail(3, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)');
-
-print STDERR "Wait until the end of replication ..\n";
-$lag = scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2',
-														   report_it := true,
-														   timeout := '10 minutes',
-														   delay := 1.)");
-ok($lag  <= 0, "Replication N2 => N1 has been finished successfully");
-$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1',
-														   report_it := true,
-														   timeout := '10 minutes',
-														   delay := 1.)");
-ok($lag  <= 0, "Replication N1 => N2 has been finished successfully");
-$lag = scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n1',
-														   report_it := true,
-														   timeout := '10 minutes',
-														   delay := 1.)");
-ok($lag  <= 0, "Replication N1 => N3 has been finished successfully");
-$lag = scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n2',
-														   report_it := true,
-														   timeout := '10 minutes',
-														   delay := 1.)");
-ok($lag  <= 0, "Replication N2 => N3 has been finished successfully");
+#
+# Wait for the end of apply process
+#
+print STDERR "Wait for the end of LR caused by the pgbench load\n";
+$lsn1 = scalar_query(1, "SELECT spock.sync_event()");
+$lsn2 = scalar_query(2, "SELECT spock.sync_event()");
+$lsn3 = scalar_query(3, "SELECT spock.sync_event()");
+print STDERR "DEBUGGING. LSNs: N1: $lsn1, N2: $lsn2, N3: $lsn3\n";
+
+print STDERR "Wait for the N2 -> N1 sync message ...\n";
+psql_or_bail(1, "CALL spock.wait_for_sync_event(true, 'n2', '$lsn2'::pg_lsn, 600)");
+print STDERR "Wait for the N1 -> N2 sync message ...\n";
+psql_or_bail(2, "CALL spock.wait_for_sync_event(true, 'n1', '$lsn1'::pg_lsn, 600)");
+print STDERR "Wait for the N1 -> N3 sync message ...\n";
+psql_or_bail(3, "CALL spock.wait_for_sync_event(true, 'n1', '$lsn1'::pg_lsn, 600)");
+print STDERR "Wait for the N2 -> N3 sync message ...\n";
+psql_or_bail(3, "CALL spock.wait_for_sync_event(true, 'n2', '$lsn2'::pg_lsn, 600)");
+print STDERR "LR messages from active nodes has arrived to the new one\n";
+
+print STDERR "Wait for the N3 -> N1 sync message ...\n";
+psql_or_bail(1, "CALL spock.wait_for_sync_event(true, 'n3', '$lsn3'::pg_lsn, 600)");
+print STDERR "Wait for the N3 -> N2 sync message ...\n";
+psql_or_bail(2, "CALL spock.wait_for_sync_event(true, 'n3', '$lsn3'::pg_lsn, 600)");
+print STDERR "First LR transaction has arrived from new node to the active ones\n";
 
 print STDERR "Check the data consistency.\n";
 $ret1 = scalar_query(1, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts");