ci: add stackprof (#289)

Author: supercaracal

.github/workflows/test.yaml

@@ -201,6 +201,36 @@ jobs:
           do
             docker compose -f $DOCKER_COMPOSE_FILE exec node$i tc qdisc del dev eth0 root netem || true
           done
+      - name: Stop containers
+        run: docker compose -f $DOCKER_COMPOSE_FILE down || true
+  ips:
+    name: IPS
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    env:
+      REDIS_VERSION: '7.2'
+      DOCKER_COMPOSE_FILE: 'compose.latency.yaml'
+      REDIS_REPLICA_SIZE: '2'
+      REDIS_CLIENT_MAX_THREADS: '10'
+      DELAY_TIME: '0ms'
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2'
+          bundler-cache: true
+      - name: Pull Docker images
+        run: docker pull redis:$REDIS_VERSION
+      - name: Run containers
+        run: docker compose -f $DOCKER_COMPOSE_FILE up -d
+      - name: Wait for Redis cluster to be ready
+        run: bundle exec rake wait
+      - name: Print containers
+        run: docker compose -f $DOCKER_COMPOSE_FILE ps
+      - name: Print cpu info
+        run: grep 'model name' /proc/cpuinfo
       - name: Run iteration per second
         run: bundle exec rake ips
       - name: Stop containers
@@ -235,10 +265,10 @@ jobs:
         run: bundle exec rake wait
       - name: Print containers
         run: docker compose -f $DOCKER_COMPOSE_FILE ps
-      - name: Run memory profiler
+      - name: Run profiler
         run: bundle exec rake prof
         env:
-          MEMORY_PROFILE_MODE: ${{ matrix.mode }}
+          PROFILE_MODE: ${{ matrix.mode }}
       - name: Stop containers
         run: docker compose -f $DOCKER_COMPOSE_FILE down || true
   massive:
@@ -295,9 +325,9 @@ jobs:
           DEBUG: '1'
       - name: Print containers
         run: docker compose -f $DOCKER_COMPOSE_FILE ps
-      - name: Run memory profiler
+      - name: Run profiler
         run: bundle exec rake prof
         env:
-          MEMORY_PROFILE_MODE: pipelining_in_moderation
+          PROFILE_MODE: pipelining_in_moderation
       - name: Stop containers
         run: docker compose -f $DOCKER_COMPOSE_FILE down || true

Gemfile

@@ -12,3 +12,4 @@ gem 'rubocop'
 gem 'rubocop-minitest', require: false
 gem 'rubocop-performance', require: false
 gem 'rubocop-rake', require: false
+gem 'stackprof', platform: :mri

Rakefile

@@ -29,28 +29,14 @@ SLUGGISH_TEST_TYPES.each do |type|
   end
 end
 
-Rake::TestTask.new(:bench) do |t|
-  t.libs << :lib
-  t.libs << :test
-  t.options = '-v'
-  t.warning = false
-  t.test_files = ARGV.size > 1 ? ARGV[1..] : Dir['test/**/bench_*.rb']
-end
-
-Rake::TestTask.new(:ips) do |t|
-  t.libs << :lib
-  t.libs << :test
-  t.options = '-v'
-  t.warning = false
-  t.test_files = ARGV.size > 1 ? ARGV[1..] : Dir['test/**/ips_*.rb']
-end
-
-Rake::TestTask.new(:prof) do |t|
-  t.libs << :lib
-  t.libs << :test
-  t.options = '-v'
-  t.warning = false
-  t.test_files = ARGV.size > 1 ? ARGV[1..] : Dir['test/**/prof_*.rb']
+%i[bench ips prof].each do |k|
+  Rake::TestTask.new(k) do |t|
+    t.libs << :lib
+    t.libs << :test
+    t.options = '-v'
+    t.warning = false
+    t.test_files = ARGV.size > 1 ? ARGV[1..] : Dir["test/**/#{k}_*.rb"]
+  end
 end
 
 desc 'Wait for cluster to be ready'

compose.latency.yaml

@@ -37,7 +37,7 @@ services:
       bash -c "apt-get update > /dev/null
       && apt-get install --no-install-recommends --no-install-suggests -y iproute2 iputils-ping > /dev/null
       && rm -rf /var/lib/apt/lists/*
-      && (tc qdisc add dev eth0 root netem delay ${DELAY_TIME:-20ms} || echo skipped)
+      && (tc qdisc add dev eth0 root netem delay ${DELAY_TIME:-1ms} || echo skipped)
       && redis-server
       --maxmemory            64mb
       --maxmemory-policy     allkeys-lru

test/prof_mem.rb

@@ -42,7 +42,7 @@ module ProfMem
   }.freeze
 
   def run
-    mode = ENV.fetch('MEMORY_PROFILE_MODE', :single).to_sym
+    mode = ENV.fetch('PROFILE_MODE', :single).to_sym
     subject = MODES.fetch(mode)
 
     CLI_TYPES.each do |cli_type|

test/prof_stack.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'tmpdir'
+require 'stackprof'
+require 'redis_cluster_client'
+require 'testing_constants'
+
+module ProfStack
+  SIZE = 40
+  ATTEMPTS = 1000
+
+  module_function
+
+  def run
+    client = make_client
+    mode = ENV.fetch('PROFILE_MODE', :single).to_sym
+    prepare(client)
+    profile = StackProf.run(mode: :cpu, raw: true) { execute(client, mode) }
+    StackProf::Report.new(profile).print_text(false, 40)
+  end
+
+  def make_client
+    ::RedisClient.cluster(
+      nodes: TEST_NODE_URIS,
+      replica: true,
+      replica_affinity: :random,
+      fixed_hostname: TEST_FIXED_HOSTNAME,
+      **TEST_GENERIC_OPTIONS
+    ).new_client
+  end
+
+  def prepare(client)
+    ATTEMPTS.times do |i|
+      client.pipelined do |pi|
+        SIZE.times do |j|
+          n = SIZE * i + j
+          pi.call('SET', "key#{n}", "val#{n}")
+        end
+      end
+    end
+  end
+
+  def execute(client, mode)
+    case mode
+    when :single
+      (ATTEMPTS * SIZE).times { |i| client.call('GET', "key#{i}") }
+    when :excessive_pipelining
+      client.pipelined do |pi|
+        (ATTEMPTS * SIZE).times { |i| pi.call('GET', "key#{i}") }
+      end
+    when :pipelining_in_moderation
+      ATTEMPTS.times do |i|
+        client.pipelined do |pi|
+          SIZE.times do |j|
+            n = SIZE * i + j
+            pi.call('GET', "key#{n}")
+          end
+        end
+      end
+    else raise ArgumentError, mode
+    end
+  end
+end
+
+ProfStack.run