test: add a case for the recoverability from cluster down (#387)

Author: supercaracal

.github/workflows/test.yaml

@@ -40,6 +40,7 @@ jobs:
           - {ruby: 'jruby'}
           - {ruby: 'truffleruby'}
           - {task: test_cluster_broken, restart: 'no', startup: '6'}
+          - {task: test_cluster_down}
           - {redis: '8',   ruby: '3.3', compose: compose.valkey.yaml, replica: '2'}
           - {redis: '7.2', ruby: '3.2', compose: compose.auth.yaml}
           - {redis: '7.0', ruby: '3.1'}
@@ -66,17 +67,17 @@ jobs:
           ruby-version: ${{ matrix.ruby || '3.3' }}
           bundler-cache: true
       - name: Pull Docker images
-        run: docker compose -f $DOCKER_COMPOSE_FILE pull
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
       - name: Run containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE up -d
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
       - name: Wait for Redis cluster to be ready
         run: bundle exec rake wait
       - name: Print containers
         run: docker compose -f $DOCKER_COMPOSE_FILE ps
       - name: Run minitest
         run: bundle exec rake ${{ matrix.task || 'test' }}
       - name: Stop containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE down || true
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
   nat-ted-env:
     name: NAT-ted Environments
     timeout-minutes: 5
@@ -97,9 +98,9 @@ jobs:
           host_ip_addr=$(ip a | grep eth0 | grep inet | awk '{print $2}' | cut -d'/' -f1)
           echo "HOST_IP_ADDR=$host_ip_addr" >> $GITHUB_ENV
       - name: Pull Docker images
-        run: docker compose -f $DOCKER_COMPOSE_FILE pull
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
       - name: Run containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE up -d
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
         env:
           HOST_ADDR: ${{ env.HOST_IP_ADDR }}
       - name: Wait for nodes to be ready
@@ -134,7 +135,7 @@ jobs:
       - name: Run minitest
         run: bundle exec rake test
       - name: Stop containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE down || true
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
   lint:
     name: Lint
     timeout-minutes: 5
@@ -168,9 +169,9 @@ jobs:
           ruby-version: '3.3'
           bundler-cache: true
       - name: Pull Docker images
-        run: docker compose -f $DOCKER_COMPOSE_FILE pull
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
       - name: Run containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE up -d
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
       - name: Wait for Redis cluster to be ready
         run: bundle exec rake wait
       - name: Print containers
@@ -208,7 +209,7 @@ jobs:
             docker compose -f $DOCKER_COMPOSE_FILE exec node$i tc qdisc del dev eth0 root netem || true
           done
       - name: Stop containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE down || true
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
   ips:
     name: IPS
     timeout-minutes: 10
@@ -228,9 +229,9 @@ jobs:
           ruby-version: '3.3'
           bundler-cache: true
       - name: Pull Docker images
-        run: docker compose -f $DOCKER_COMPOSE_FILE pull
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
       - name: Run containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE up -d
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
       - name: Wait for Redis cluster to be ready
         run: bundle exec rake wait
       - name: Print containers
@@ -240,7 +241,7 @@ jobs:
       - name: Run iteration per second
         run: bundle exec rake ips
       - name: Stop containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE down || true
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
   profiling:
     name: Profiling
     timeout-minutes: 5
@@ -266,9 +267,9 @@ jobs:
           ruby-version: '3.3'
           bundler-cache: true
       - name: Pull Docker images
-        run: docker compose -f $DOCKER_COMPOSE_FILE pull
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
       - name: Run containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE up -d
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
       - name: Wait for Redis cluster to be ready
         run: bundle exec rake wait
       - name: Print containers
@@ -278,7 +279,7 @@ jobs:
         env:
           PROFILE_MODE: ${{ matrix.mode }}
       - name: Stop containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE down || true
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
   massive:
     name: Massive Cluster
     timeout-minutes: 10
@@ -322,9 +323,9 @@ jobs:
           sudo sysctl -w net.ipv4.tcp_max_syn_backlog=1024 # backlog setting
           sudo sysctl -w net.core.somaxconn=1024           # up the number of connections per port
       - name: Pull Docker images
-        run: docker compose -f $DOCKER_COMPOSE_FILE pull
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
       - name: Run containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE up -d
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
       - name: Print memory info
         run: free -w
       - name: Wait for Redis cluster to be ready
@@ -338,4 +339,4 @@ jobs:
         env:
           PROFILE_MODE: pipelining_in_moderation
       - name: Stop containers
-        run: docker compose -f $DOCKER_COMPOSE_FILE down || true
+        run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true

Rakefile

@@ -7,7 +7,7 @@ require 'bundler/gem_helper'
 RuboCop::RakeTask.new
 Bundler::GemHelper.install_tasks
 
-SLUGGISH_TEST_TYPES = %w[broken scale state].freeze
+SLUGGISH_TEST_TYPES = %w[down broken scale state].freeze
 
 task default: :test
 

bin/pubsub

@@ -53,8 +53,9 @@ module PubSubDebug
       ensure
         sleep 1.0
       end
-    rescue StandardError => e
+    rescue StandardError, SignalException => e
       log "#{role}: dead: #{e.class}: #{e.message}"
+      ps&.close
       raise
     end
   end

bin/singlepiptx

@@ -50,10 +50,12 @@ module SinglePipTxDebug
   def spawn_transaction(cli)
     Thread.new(cli) do |r|
       role = 'Transaction'
+      i = 0
 
       loop do
         handle_errors(role) do
-          reply = r.multi do |tx|
+          reply = r.multi(watch: i.odd? ? %w[transaction] : nil) do |tx|
+            i += 1
             tx.call('incr', 'transaction')
             tx.call('incr', 'transaction')
             tx.call('incr', 'transaction')